Update dbcopilot version (#1671)

* Update dbcopilot version * add docstring * update dbcopilot_tool version
Azure · Nov 6, 2023 · 0b18f02 · 0b18f02
1 parent 22c0ee9
commit 0b18f02
Show file tree

Hide file tree

Showing 14 changed files with 38 additions and 27 deletions.
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml
@@ -4,7 +4,7 @@ tags:
   Preview: ""
 name: llm_ingest_db_to_acs
 display_name: LLM - SQL Datastore to ACS Pipeline
-version: 0.0.39
+version: 0.0.40
 description: Single job pipeline to chunk data from AzureML sql data store, and create ACS embeddings index
 settings:
   default_compute: serverless
@@ -107,7 +107,7 @@ jobs:
       output_chunk_file:
         type: uri_folder
       output_grounding_context_file: ${{parent.outputs.db_context}}
-    component: "azureml:llm_dbcopilot_grounding:0.0.31"
+    component: "azureml:llm_dbcopilot_grounding:0.0.32"
     type: command
   generate_meta_embeddings:
     type: command
@@ -213,5 +213,5 @@ jobs:
         path: ${{parent.inputs.chat_aoai_deployment_name}}
       llm_config:
         path: ${{parent.inputs.llm_config}}
-    component: "azureml:llm_dbcopilot_create_promptflow:0.0.31"
+    component: "azureml:llm_dbcopilot_create_promptflow:0.0.32"
     type: command
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml
@@ -4,7 +4,7 @@ tags:
   Preview: ""
 name: llm_ingest_db_to_faiss
 display_name: LLM - SQL Datastore to FAISS Pipeline
-version: 0.0.39
+version: 0.0.40
 description: Single job pipeline to chunk data from AzureML sql data store, and create FAISS embeddings index
 settings:
   default_compute: serverless
@@ -100,7 +100,7 @@ jobs:
       output_chunk_file:
         type: uri_folder
       output_grounding_context_file: ${{parent.outputs.db_context}}
-    component: "azureml:llm_dbcopilot_grounding:0.0.31"
+    component: "azureml:llm_dbcopilot_grounding:0.0.32"
     type: command
   generate_meta_embeddings:
     type: command
@@ -203,5 +203,5 @@ jobs:
         path: ${{parent.inputs.chat_aoai_deployment_name}}
       llm_config:
         path: ${{parent.inputs.llm_config}}
-    component: "azureml:llm_dbcopilot_create_promptflow:0.0.31"
+    component: "azureml:llm_dbcopilot_create_promptflow:0.0.32"
     type: command
diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml
@@ -2,7 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.js
 type: pipeline
 
 name: llm_ingest_dbcopilot_acs_e2e
-version: 0.0.10
+version: 0.0.11
 display_name: Data Ingestion for DB Data Output to ACS E2E Deployment
 description: Single job pipeline to chunk data from AzureML DB Datastore and create acs embeddings index
 
@@ -120,7 +120,7 @@ jobs:
   #########################################
   db_meta_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding:0.0.31"
+    component: "azureml:llm_dbcopilot_grounding:0.0.32"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -188,7 +188,7 @@ jobs:
   #########################################
   db_sample_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.7"
+    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.8"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -250,7 +250,7 @@ jobs:
   #########################################
   endpoint_deployment_job:
     type: command
-    component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.7"
+    component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.8"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}

diff --git a/...s/large_language_models/components_pipelines/data_ingestion_dbcopilot_faiss_e2e/spec.yaml b/...s/large_language_models/components_pipelines/data_ingestion_dbcopilot_faiss_e2e/spec.yaml
@@ -2,7 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/pipelineComponent.schema.js
 type: pipeline
 
 name: llm_ingest_dbcopilot_faiss_e2e
-version: 0.0.10
+version: 0.0.11
 display_name: Data Ingestion for DB Data Output to FAISS E2E Deployment
 description: Single job pipeline to chunk data from AzureML DB Datastore and create faiss embeddings index
 
@@ -110,7 +110,7 @@ jobs:
   #########################################
   db_meta_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding:0.0.31"
+    component: "azureml:llm_dbcopilot_grounding:0.0.32"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -176,7 +176,7 @@ jobs:
   #########################################
   db_sample_loading_generator:
     type: command
-    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.7"
+    component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.8"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}
@@ -234,7 +234,7 @@ jobs:
   #########################################
   endpoint_deployment_job:
     type: command
-    component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.7"
+    component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.8"
     resources:
       instance_count: ${{parent.inputs.serverless_instance_count}}
       instance_type: ${{parent.inputs.serverless_instance_type}}

diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_create_promptflow/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_create_promptflow/spec.yaml
@@ -4,7 +4,7 @@ tags:
   Preview: ""
 name: llm_dbcopilot_create_promptflow
 display_name: LLM - Create DBCopilot Prompt Flow
-version: 0.0.31
+version: 0.0.32
 inputs:
   index_name:
     type: string

diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_deploy_endpoint/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_deploy_endpoint/spec.yaml
@@ -3,7 +3,7 @@ type: command
 tags: {}
 name: llm_dbcopilot_deploy_endpoint
 display_name: DBCopilot Deploy Endpoint Component
-version: 0.0.7
+version: 0.0.8
 inputs:
   deployment_name:
     type: string

diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding/spec.yaml
@@ -4,7 +4,7 @@ tags:
   Preview: ""
 name: llm_dbcopilot_grounding
 display_name: LLM - DBCopilot Grounding DB
-version: 0.0.31
+version: 0.0.32
 inputs:
   asset_uri:
     type: string

diff --git a/...rge_language_models/dbcopilot/components/llm_dbcopilot_grounding_ground_samples/spec.yaml b/...rge_language_models/dbcopilot/components/llm_dbcopilot_grounding_ground_samples/spec.yaml
@@ -3,7 +3,7 @@ type: command
 tags: {}
 name: llm_dbcopilot_grounding_ground_samples
 display_name: DBCopilot Grounding Ground Samples Component
-version: 0.0.7
+version: 0.0.8
 inputs:
   grounding_context:
     type: uri_folder

diff --git a/assets/large_language_models/dbcopilot/components/src/db_copilot_mir/code/score_zero.py b/assets/large_language_models/dbcopilot/components/src/db_copilot_mir/code/score_zero.py
@@ -126,10 +126,19 @@ def set_db_copilot_adapter(db_name, db_copilot_config, visibility):
     """Set a dbcopilot adapter."""
     cache_dir = os.getenv("AZUREML_MODEL_DIR")
     cache_uri = os.getenv("DBCOPILOT_CACHE_URI")
+
+    def simple_db_name(db_name):
+        if db_name.startswith("azureml://"):
+            return db_name.split("/")[-1].replace(".", "_")
+        return db_name
+
     if cache_uri:
-        cache_folder = os.path.join(cache_dir, visibility.value.lower(), db_name)
+        cache_folder = os.path.join(
+            cache_dir, visibility.value.lower(), simple_db_name(db_name)
+        )
         with DatastoreUploader(
-            f"{cache_uri}{visibility.value.lower()}/{db_name}", cache_folder
+            f"{cache_uri}{visibility.value.lower()}/{simple_db_name(db_name)}",
+            cache_folder,
         ):
             db_copilot_adapter = DBCopilotAdapter(
                 db_copilot_config,

diff --git a/assets/large_language_models/dbcopilot/components/src/endpoint_deployment_base.py b/assets/large_language_models/dbcopilot/components/src/endpoint_deployment_base.py
@@ -49,7 +49,9 @@ def get_secrets(self):
                 "embedding": embedding_connection_id,
                 "chat": chat_connection_id,
             }.items():
-                connection = get_connection_by_id_v2(connection_id)
+                connection = get_connection_by_id_v2(
+                    connection_id, credential=self.mlclient_credential
+                )
                 credential = connection_to_credential(connection)
                 if hasattr(credential, "key"):
                     secrets.update(

diff --git a/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/Dockerfile b/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/Dockerfile
@@ -29,7 +29,7 @@ COPY ./extra_requirements.txt ./
 RUN pip install -r extra_requirements.txt
 
 ## Install dbcopilot
-RUN pip install https://ragsample.blob.core.windows.net/ragdata/wheels/dbcopilot/db_copilot_tool-0.1.9-py3-none-any.whl
+RUN pip install https://ragsample.blob.core.windows.net/ragdata/wheels/dbcopilot/db_copilot_tool-0.1.10-py3-none-any.whl
 
 # fix for https://portal.microsofticm.com/imp/v3/incidents/incident/418102604/summary
 RUN pip install cryptography==41.0.4

diff --git a/...anguage_models/dbcopilot/environments/dbcopilot_embeddings/context/extra_requirements.txt b/...anguage_models/dbcopilot/environments/dbcopilot_embeddings/context/extra_requirements.txt
@@ -1,3 +1,3 @@
-promptflow[azure]==0.1.0b7.post1
-promptflow-tools==0.1.0b6
+promptflow[azure]==0.1.0b8
+promptflow-tools==0.1.0b10
 promptflow-vectordb==0.1.2
diff --git a/assets/large_language_models/dbcopilot/environments/dbcopilot_mir/context/Dockerfile b/assets/large_language_models/dbcopilot/environments/dbcopilot_mir/context/Dockerfile
@@ -30,7 +30,7 @@ COPY ./extra_requirements.txt ./
 RUN pip install -r extra_requirements.txt
 
 ## Install dbcopilot
-RUN pip install https://ragsample.blob.core.windows.net/ragdata/wheels/dbcopilot/db_copilot_tool-0.1.9-py3-none-any.whl
+RUN pip install https://ragsample.blob.core.windows.net/ragdata/wheels/dbcopilot/db_copilot_tool-0.1.10-py3-none-any.whl
 
 RUN pip install cryptography==41.0.4 langchain==0.0.236
 

diff --git a/...large_language_models/dbcopilot/environments/dbcopilot_mir/context/extra_requirements.txt b/...large_language_models/dbcopilot/environments/dbcopilot_mir/context/extra_requirements.txt
@@ -1,3 +1,3 @@
-promptflow[azure]==0.1.0b7.post1
-promptflow-tools==0.1.0b6
+promptflow[azure]==0.1.0b8
+promptflow-tools==0.1.0b10
 promptflow-vectordb==0.1.2