Support fine-grained provisioning; handle more edge cases on search flow

common/interfaces.ts

@@ -58,15 +58,25 @@ export type IndexConfig = {
   settings: IConfigField;
 };
 
+// TODO: may expand to just IndexConfig (including mappings/settings info)
+// if we want to persist this for users using some existing index,
+// and want to pass that index config around.
+export type SearchIndexConfig = {
+  name: IConfigField;
+};
+
 export type IngestConfig = {
-  enabled: boolean;
+  enabled: IConfigField;
   source: {};
+  pipelineName: IConfigField;
   enrich: ProcessorsConfig;
   index: IndexConfig;
 };
 
 export type SearchConfig = {
   request: IConfigField;
+  index: SearchIndexConfig;
+  pipelineName: IConfigField;
   enrichRequest: ProcessorsConfig;
   enrichResponse: ProcessorsConfig;
 };

public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx

@@ -234,7 +234,11 @@ export function InputTransformModal(props: InputTransformModalProps) {
                 helpText={`An array specifying how to map fields from the ingested document to the model’s input.`}
                 helpLink={ML_INFERENCE_DOCS_LINK}
                 keyPlaceholder="Model input field"
-                valuePlaceholder="Document field"
+                valuePlaceholder={
+                  props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
+                    ? 'Query field'
+                    : 'Document field'
+                }
                 keyOptions={props.inputFields}
                 onFormChange={props.onFormChange}
                 // If the map we are adding is the first one, populate the selected option to index 0

public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs.tsx

@@ -229,7 +229,11 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
             helpText={`An array specifying how to map fields from the ingested document to the model’s input.`}
             helpLink={ML_INFERENCE_DOCS_LINK}
             keyPlaceholder="Model input field"
-            valuePlaceholder="Document field"
+            valuePlaceholder={
+              props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
+                ? 'Query field'
+                : 'Document field'
+            }
             onFormChange={props.onFormChange}
             keyOptions={inputFields}
           />
@@ -273,7 +277,11 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) {
             label="Output Map"
             helpText={`An array specifying how to map the model’s output to new document fields.`}
             helpLink={ML_INFERENCE_DOCS_LINK}
-            keyPlaceholder="New document field"
+            keyPlaceholder={
+              props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST
+                ? 'Query field'
+                : 'Document field'
+            }
             valuePlaceholder="Model output field"
             onFormChange={props.onFormChange}
             valueOptions={outputFields}

public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx

@@ -108,7 +108,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
                       const valuesWithoutOutputMapConfig = cloneDeep(values);
                       set(
                         valuesWithoutOutputMapConfig,
-                        `ingest.enrich.${props.config.id}.outputMap`,
+                        `ingest.enrich.${props.config.id}.output_map`,
                         []
                       );
                       const curIngestPipeline = formikToPartialPipeline(
@@ -146,7 +146,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
                       const valuesWithoutOutputMapConfig = cloneDeep(values);
                       set(
                         valuesWithoutOutputMapConfig,
-                        `search.enrichResponse.${props.config.id}.outputMap`,
+                        `search.enrichResponse.${props.config.id}.output_map`,
                         []
                       );
                       const curSearchPipeline = formikToPartialPipeline(
@@ -226,7 +226,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
                 label="Output Map"
                 helpText={`An array specifying how to map the model’s output to new fields.`}
                 helpLink={ML_INFERENCE_DOCS_LINK}
-                keyPlaceholder="New document field"
+                keyPlaceholder="Document field"
                 valuePlaceholder="Model output field"
                 valueOptions={props.outputFields}
                 onFormChange={props.onFormChange}

public/pages/workflow_detail/workflow_inputs/search_inputs/configure_search_request.tsx

@@ -22,7 +22,7 @@ import {
   EuiText,
   EuiTitle,
 } from '@elastic/eui';
-import { SearchHit, WorkspaceFormValues } from '../../../../../common';
+import { SearchHit, WorkflowFormValues } from '../../../../../common';
 import { JsonField } from '../input_fields';
 import {
   AppState,
@@ -44,18 +44,27 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
   const dispatch = useAppDispatch();
 
   // Form state
-  const { values } = useFormikContext<WorkspaceFormValues>();
-  const indexName = values.ingest.index.name;
+  const { values, setFieldValue, setFieldTouched } = useFormikContext<
+    WorkflowFormValues
+  >();
   const ingestEnabled = values.ingest.enabled;
+  const searchIndexNameFormPath = 'search.index.name';
 
   // All indices state
   const indices = useSelector((state: AppState) => state.opensearch.indices);
 
   // Selected index state
   const [selectedIndex, setSelectedIndex] = useState<string | undefined>(
-    undefined
+    values.search.index.name
   );
 
+  // initial load: set the search index value, if not already set
+  useEffect(() => {
+    if (values.ingest.enabled) {
+      setFieldValue(searchIndexNameFormPath, values.ingest.index.name);
+    }
+  }, []);
+
   // Edit modal state
   const [isEditModalOpen, setIsEditModalOpen] = useState<boolean>(false);
 
@@ -116,7 +125,7 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
         <EuiFlexItem grow={false}>
           <EuiFormRow label="Retrieval index">
             {ingestEnabled ? (
-              <EuiFieldText value={indexName} readOnly={true} />
+              <EuiFieldText value={values.ingest.index.name} readOnly={true} />
             ) : (
               <EuiSuperSelect
                 options={Object.values(indices).map(
@@ -130,6 +139,9 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
                 valueOfSelected={selectedIndex}
                 onChange={(option) => {
                   setSelectedIndex(option);
+                  setFieldValue(searchIndexNameFormPath, option);
+                  setFieldTouched(searchIndexNameFormPath, true);
+                  props.onFormChange();
                 }}
                 isInvalid={selectedIndex === undefined}
               />
@@ -165,7 +177,7 @@ export function ConfigureSearchRequest(props: ConfigureSearchRequestProps) {
               // see https://opensearch.org/docs/latest/search-plugins/search-pipelines/using-search-pipeline/#disabling-the-default-pipeline-for-a-request
               dispatch(
                 searchIndex({
-                  index: indexName,
+                  index: values.search.index.name,
                   body: values.search.request,
                   searchPipeline: '_none',
                 })