adding state index initial

Signed-off-by: Amit Galitzky <[email protected]>
opensearch-project · Oct 20, 2023 · 9469bf1 · 9469bf1
1 parent 9784f41
commit 9469bf1
Show file tree

Hide file tree

Showing 24 changed files with 1,440 additions and 607 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -1,4 +1,4 @@
-name: Build and Test Plugin
+name: CI
 
 on:
   push:
@@ -28,16 +28,14 @@ jobs:
     needs: [spotless, javadoc]
     strategy:
       matrix:
-        os:
-          - ubuntu-latest
-          - macOS-latest
-          - windows-latest
-        java:
-          - 17
-    name: Build and Test Plugin Template
-    if: github.repository == 'opensearch-project/opensearch-ai-flow-framework'
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        java: [11, 20]
+        include:
+          - os: ubuntu-latest
+            java: 17
+            codecov: yes
+    name: Test JDK${{ matrix.java }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-
     steps:
     - uses: actions/checkout@v4
     - name: Set up JDK ${{ matrix.java }}
@@ -47,9 +45,30 @@ jobs:
         distribution: temurin
     - name: Build and Run Tests
       run: |
-          ./gradlew check
+          ./gradlew check -x integTest -x yamlRestTest
     - name: Upload Coverage Report
-      if: matrix.os == 'ubuntu-latest'
+      if: ${{ matrix.codecov }}
       uses: codecov/codecov-action@v3
       with:
         file: ./build/reports/jacoco/test/jacocoTestReport.xml
+  integTest:
+    needs: [spotless, javadoc]
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        java: [11, 20]
+        include:
+          - os: ubuntu-latest
+            java: 17
+    name: Integ Test JDK${{ matrix.java }}, ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{ matrix.java }}
+        distribution: temurin
+    - name: Build and Run Tests
+      run: |
+          ./gradlew integTest yamlRestTest
diff --git a/build.gradle b/build.gradle
@@ -56,6 +56,7 @@ buildscript {
         opensearch_group = "org.opensearch"
         opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
         System.setProperty('tests.security.manager', 'false')
+        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
     }
 
     repositories {
@@ -135,6 +136,7 @@ dependencies {
     implementation 'org.junit.jupiter:junit-jupiter:5.10.0'
     implementation "com.google.guava:guava:32.1.3-jre"
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
+    implementation "org.opensearch:common-utils:${common_utils_version}"
 
     configurations.all {
         resolutionStrategy {

diff --git a/src/main/java/org/opensearch/flowframework/FlowFrameworkPlugin.java b/src/main/java/org/opensearch/flowframework/FlowFrameworkPlugin.java
@@ -24,14 +24,13 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
-import org.opensearch.flowframework.indices.GlobalContextHandler;
+import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
 import org.opensearch.flowframework.rest.RestCreateWorkflowAction;
 import org.opensearch.flowframework.rest.RestProvisionWorkflowAction;
 import org.opensearch.flowframework.transport.CreateWorkflowAction;
 import org.opensearch.flowframework.transport.CreateWorkflowTransportAction;
 import org.opensearch.flowframework.transport.ProvisionWorkflowAction;
 import org.opensearch.flowframework.transport.ProvisionWorkflowTransportAction;
-import org.opensearch.flowframework.workflow.CreateIndexStep;
 import org.opensearch.flowframework.workflow.WorkflowProcessSorter;
 import org.opensearch.flowframework.workflow.WorkflowStepFactory;
 import org.opensearch.plugins.ActionPlugin;
@@ -79,10 +78,9 @@ public Collection<Object> createComponents(
         WorkflowStepFactory workflowStepFactory = new WorkflowStepFactory(clusterService, client);
         WorkflowProcessSorter workflowProcessSorter = new WorkflowProcessSorter(workflowStepFactory, threadPool);
 
-        // TODO : Refactor, move system index creation/associated methods outside of the CreateIndexStep
-        GlobalContextHandler globalContextHandler = new GlobalContextHandler(client, new CreateIndexStep(clusterService, client));
+        FlowFrameworkIndicesHandler flowFrameworkIndicesHandler = new FlowFrameworkIndicesHandler(client, clusterService);
 
-        return ImmutableList.of(workflowStepFactory, workflowProcessSorter, globalContextHandler);
+        return ImmutableList.of(workflowStepFactory, workflowProcessSorter, flowFrameworkIndicesHandler);
     }
 
     @Override

diff --git a/src/main/java/org/opensearch/flowframework/common/CommonValue.java b/src/main/java/org/opensearch/flowframework/common/CommonValue.java
@@ -27,6 +27,12 @@ private CommonValue() {}
     public static final String GLOBAL_CONTEXT_INDEX_MAPPING = "mappings/global-context.json";
     /** Global Context index mapping version */
     public static final Integer GLOBAL_CONTEXT_INDEX_VERSION = 1;
+    /** Workflow State Index Name */
+    public static final String WORKFLOW_STATE_INDEX = ".plugins-workflow-state";
+    /** Workflow State index mapping file path */
+    public static final String WORKFLOW_STATE_INDEX_MAPPING = "mappings/workflow-state.json";
+    /** Workflow State index mapping version */
+    public static final Integer WORKFLOW_STATE_INDEX_VERSION = 1;
 
     /** The transport action name prefix */
     public static final String TRANSPORT_ACION_NAME_PREFIX = "cluster:admin/opensearch/flow_framework/";

diff --git a/src/main/java/org/opensearch/flowframework/indices/FlowFrameworkIndex.java b/src/main/java/org/opensearch/flowframework/indices/FlowFrameworkIndex.java
@@ -14,6 +14,8 @@
 
 import static org.opensearch.flowframework.common.CommonValue.GLOBAL_CONTEXT_INDEX;
 import static org.opensearch.flowframework.common.CommonValue.GLOBAL_CONTEXT_INDEX_VERSION;
+import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_STATE_INDEX;
+import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_STATE_INDEX_VERSION;
 
 /**
  * An enumeration of Flow Framework indices
@@ -24,8 +26,13 @@ public enum FlowFrameworkIndex {
     */
     GLOBAL_CONTEXT(
         GLOBAL_CONTEXT_INDEX,
-        ThrowingSupplierWrapper.throwingSupplierWrapper(GlobalContextHandler::getGlobalContextMappings),
+        ThrowingSupplierWrapper.throwingSupplierWrapper(FlowFrameworkIndicesHandler::getGlobalContextMappings),
         GLOBAL_CONTEXT_INDEX_VERSION
+    ),
+    WORKFLOW_STATE(
+        WORKFLOW_STATE_INDEX,
+        ThrowingSupplierWrapper.throwingSupplierWrapper(FlowFrameworkIndicesHandler::getGlobalContextMappings),
+        WORKFLOW_STATE_INDEX_VERSION
     );
 
     private final String indexName;