From 5a552098659707e89c30cef18eeacffbe1269290 Mon Sep 17 00:00:00 2001 From: mreyesgomez Date: Wed, 2 Oct 2024 11:27:56 -0700 Subject: [PATCH 1/4] Adding support for optional creation of Resource Group and Container Registry --- .../azure/azureml/cli/1_set_credentials.sh | 22 ++++++++++++++++++- .../azure/azureml/cli/README.md | 10 +++++---- .../azure/azureml/cli/config.sh | 18 ++++++++------- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh b/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh index 83ba82a..8ced440 100755 --- a/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh +++ b/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh @@ -2,18 +2,38 @@ set -x source config.sh +CREATE_RESOURCE_GROUP=false +CREATE_CONTAINER_REGISTRY=false CREATE_WORKSPACE=false for i in "$@"; do + case $i in + --create_new_resource) CREATE_RESOURCE_GROUP=true ;; + -*|--*) echo "Unknown option $i"; exit 1 ;; + esac + case $i in + --create_new_container_registry) CREATE_CONTAINER_REGISTRY=true ;; + -*|--*) echo "Unknown option $i"; exit 1 ;; + esac case $i in --create_new_workspace) CREATE_WORKSPACE=true ;; -*|--*) echo "Unknown option $i"; exit 1 ;; esac done +# Create new resource group +if $CREATE_RESOURCE_GROUP then + az group create --name $resource_group --location $location +fi + +# Create new container registry +if $CREATE_CONTAINER_REGISTRY then + az acr create --resource-group $resource_group --name $acr_registry_name --sku Basic +fi + # Create new workspace if $CREATE_WORKSPACE; then - az ml workspace create --name $workspace --resource-group $resource_group --location $location + az ml workspace create --name $workspace --resource-group $resource_group --location $location --container-registry /subscriptions/${subscription_id}/resourceGroups/${resource_group}/providers/Microsoft.ContainerRegistry/registries/${acr_registry_name} fi # Assign role permission to read secrets from workspace connections diff --git a/cloud-service-providers/azure/azureml/cli/README.md b/cloud-service-providers/azure/azureml/cli/README.md index a871913..4e019a5 100755 --- a/cloud-service-providers/azure/azureml/cli/README.md +++ b/cloud-service-providers/azure/azureml/cli/README.md @@ -27,14 +27,16 @@ az account set -s ${subscription_id} ## Setup AzureML Workspace -Create a new AzureML workspace with the "Azure ML Secrets Reader" role assignment. All necessary commands are provided in the `1_setup_credentials.sh` script file. +The deployment procedure requires an Azure AI Workspace with an associated Azure AI private container registry. The workspace will also require to assign the "Azure ML Secrets Reader" role assignment to the user. The user could use any pre-existent workspace with those characteristics, populating accordingly the names of the resource group, container registry name and workspace name in the config.sh file. + +The `1_setup_credentials.sh` script file, provides the role assigment, optionally it can create the necessary resource group, container registry and workspace (with the required association to the container registry, all using the names provided in the config.sh file) by the use of flags `--create_new_resource`, `--create_new_container_registry` and `--create_new_workspace` respectively. + +Create a new AzureML resource group, container registry and workspace (if needed) with the "Azure ML Secrets Reader" role assignment. All necessary commands are provided in the `1_setup_credentials.sh` script file. ```bash -./1_setup_credentials.sh --create_new_workspace +./1_setup_credentials.sh --create_new_resource --create_new_container_registry --create_new_workspace ``` -The above command creates a new workspace with the workspace name provided in the `config.sh` file. If the workspace name already exists, omit the `--create_new_workspace` flag to skip the creation process and update the workspace with the necessary role assignments. - ## Store NGC API Key for Use in the AzureML Deployment The NGC API Key needs to be stored within Azure so the AzureML workspace can access it during deployment. The API key is required to pull the correct model from the NGC model catalog. The key can be stored in Azure Key Vault or provided as a workspace connection to the AzureML workspace. diff --git a/cloud-service-providers/azure/azureml/cli/config.sh b/cloud-service-providers/azure/azureml/cli/config.sh index 49cb9c6..a5c6707 100755 --- a/cloud-service-providers/azure/azureml/cli/config.sh +++ b/cloud-service-providers/azure/azureml/cli/config.sh @@ -22,20 +22,22 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # AzureML Workspace and corresponding container registry related information -subscription_id="" -resource_group="" -workspace="" -location="" # eg: "southcentralus", "westeurope" etc. +subscription_id="ab221ca4-f098-422d-ab2f-5073b3851e68" +resource_group="NIMDeploDemo_rg" +workspace="NIMDeployDemo_ws" +location="southcentralus" # Azure keyvault creation related information -ngc_api_key="" +ngc_api_key="azdwMG00YXNicWVzcnFrN3ZqMnFtcTAyb2U6NzRhZTUxODgtNzBhMS00ZDIzLThlODktZWNkNGQwMTE5OWRh" keyvault_name="NGC-Credentials" -email_address="" +email_address="mreyesgomez@nvidia.com" # Container related information # NOTE: Verify that your AML workspace can access this ACR -acr_registry_name="" -image_name="" + +acr_registry_name="nimdeploydemowsacr" + +image_name="llama3_8b_nim_ncd" ngc_container="nvcr.io/nim/meta/llama3-8b-instruct:1.0.0" # Endpoint related information From e2630630378eb5b92e8032e23b494ef9394ccc8e Mon Sep 17 00:00:00 2001 From: mreyesgomez Date: Sun, 20 Oct 2024 19:08:36 -0700 Subject: [PATCH 2/4] adding support for resource_group and conyaoiner registry creation, fixing sed commands and container label --- .../azure/azureml/cli/1_set_credentials.sh | 32 ++++++++++++------- .../azure/azureml/cli/4_create_endpoint.sh | 2 +- .../azure/azureml/cli/5_create_deployment.sh | 12 +++---- .../azure/azureml/cli/README.md | 13 ++++---- .../azureml/cli/azureml_files/deployment.yml | 2 +- .../azure/azureml/cli/config.sh | 2 +- 6 files changed, 36 insertions(+), 27 deletions(-) diff --git a/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh b/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh index 8ced440..02de6be 100755 --- a/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh +++ b/cloud-service-providers/azure/azureml/cli/1_set_credentials.sh @@ -8,26 +8,34 @@ CREATE_WORKSPACE=false for i in "$@"; do case $i in - --create_new_resource) CREATE_RESOURCE_GROUP=true ;; - -*|--*) echo "Unknown option $i"; exit 1 ;; - esac - case $i in - --create_new_container_registry) CREATE_CONTAINER_REGISTRY=true ;; - -*|--*) echo "Unknown option $i"; exit 1 ;; - esac - case $i in - --create_new_workspace) CREATE_WORKSPACE=true ;; - -*|--*) echo "Unknown option $i"; exit 1 ;; + --create_new_resource_group) + CREATE_RESOURCE_GROUP=true + shift # past argument with no value + ;; + --create_new_container_registry) + CREATE_CONTAINER_REGISTRY=true + shift # past argument with no value + ;; + --create_new_workspace) + CREATE_WORKSPACE=true + shift # past argument with no value + ;; + -*|--*) + echo "Unknown option $i" + exit 1 + ;; + *) + ;; esac done # Create new resource group -if $CREATE_RESOURCE_GROUP then +if $CREATE_RESOURCE_GROUP; then az group create --name $resource_group --location $location fi # Create new container registry -if $CREATE_CONTAINER_REGISTRY then +if $CREATE_CONTAINER_REGISTRY; then az acr create --resource-group $resource_group --name $acr_registry_name --sku Basic fi diff --git a/cloud-service-providers/azure/azureml/cli/4_create_endpoint.sh b/cloud-service-providers/azure/azureml/cli/4_create_endpoint.sh index 2d33855..693cb8f 100755 --- a/cloud-service-providers/azure/azureml/cli/4_create_endpoint.sh +++ b/cloud-service-providers/azure/azureml/cli/4_create_endpoint.sh @@ -5,7 +5,7 @@ source config.sh # Create new endpoint in this workspace cp azureml_files/endpoint.yml actual_endpoint_aml.yml # sed -i "s/endpoint_name_placeholder/${endpoint_name}/g" actual_endpoint_aml.yml -sed -i '' "s|endpoint_name_placeholder|$endpoint_name|g" actual_endpoint_aml.yml +sed -i "s|endpoint_name_placeholder|$endpoint_name|g" actual_endpoint_aml.yml echo "Creating Online Endpoint ${endpoint_name}" az ml online-endpoint create -f actual_endpoint_aml.yml --resource-group $resource_group --workspace-name $workspace rm actual_endpoint_aml.yml diff --git a/cloud-service-providers/azure/azureml/cli/5_create_deployment.sh b/cloud-service-providers/azure/azureml/cli/5_create_deployment.sh index 7f62889..c500e47 100755 --- a/cloud-service-providers/azure/azureml/cli/5_create_deployment.sh +++ b/cloud-service-providers/azure/azureml/cli/5_create_deployment.sh @@ -10,12 +10,12 @@ cp azureml_files/deployment.yml actual_deployment_aml.yml connection_path="\${{azureml://connections/ngc/credentials/NGC_API_KEY}}" # Replace placeholders in the actual_deployment_aml.yml file -sed -i '' "s|ngc_api_key_placeholder|${connection_path}|g" actual_deployment_aml.yml -sed -i '' "s|endpoint_name_placeholder|$endpoint_name|g" actual_deployment_aml.yml -sed -i '' "s|deployment_name_placeholder|$deployment_name|g" actual_deployment_aml.yml -sed -i '' "s|acr_registry_placeholder|$acr_registry_name|g" actual_deployment_aml.yml -sed -i '' "s|image_name_placeholder|$image_name|g" actual_deployment_aml.yml -sed -i '' "s|instance_type_placeholder|$instance_type|g" actual_deployment_aml.yml +sed -i "s|ngc_api_key_placeholder|${connection_path}|g" actual_deployment_aml.yml +sed -i "s|endpoint_name_placeholder|$endpoint_name|g" actual_deployment_aml.yml +sed -i "s|deployment_name_placeholder|$deployment_name|g" actual_deployment_aml.yml +sed -i "s|acr_registry_placeholder|$acr_registry_name|g" actual_deployment_aml.yml +sed -i "s|image_name_placeholder|$image_name|g" actual_deployment_aml.yml +sed -i "s|instance_type_placeholder|$instance_type|g" actual_deployment_aml.yml # Display the modified file cat actual_deployment_aml.yml diff --git a/cloud-service-providers/azure/azureml/cli/README.md b/cloud-service-providers/azure/azureml/cli/README.md index 4e019a5..dd133fa 100755 --- a/cloud-service-providers/azure/azureml/cli/README.md +++ b/cloud-service-providers/azure/azureml/cli/README.md @@ -29,7 +29,7 @@ az account set -s ${subscription_id} The deployment procedure requires an Azure AI Workspace with an associated Azure AI private container registry. The workspace will also require to assign the "Azure ML Secrets Reader" role assignment to the user. The user could use any pre-existent workspace with those characteristics, populating accordingly the names of the resource group, container registry name and workspace name in the config.sh file. -The `1_setup_credentials.sh` script file, provides the role assigment, optionally it can create the necessary resource group, container registry and workspace (with the required association to the container registry, all using the names provided in the config.sh file) by the use of flags `--create_new_resource`, `--create_new_container_registry` and `--create_new_workspace` respectively. +The `1_setup_credentials.sh` script file, provides the role assigment, optionally it can create the necessary resource group, container registry and workspace (with the required association to the container registry, all using the names provided in the config.sh file) by the use of flags `--create_new_resource_group`, `--create_new_container_registry` and `--create_new_workspace` respectively. Create a new AzureML resource group, container registry and workspace (if needed) with the "Azure ML Secrets Reader" role assignment. All necessary commands are provided in the `1_setup_credentials.sh` script file. @@ -98,9 +98,10 @@ For example: ```bash curl -X 'POST' \ - 'https://llama3-8b-nim-endpoint-aml-1.westeurope.inference.ml.azure.com/v1/chat/completions' \ + 'https://llama3-8b-nim-endpoint-aml-1.southcentralus.inference.ml.azure.com/v1/chat/completions' \ -H 'accept: application/json' \ - -H 'Authorization: Bearer xxxxxxxxxxxxxxxxxxxxxx' \ + -H 'Authorization: Bearer hNMiUaTjQGYT1FobgtzOEaYJ1ei6lPYt' \ + -H 'azureml-model-deployment: llama3-8b-nim-deployment-aml-1' \ -H 'Content-Type: application/json' \ -d '{ "messages": [ @@ -114,11 +115,11 @@ curl -X 'POST' \ } ], "model": "meta/llama3-8b-instruct", - "max_tokens": 16, + "max_tokens": 400, "top_p": 1, "n": 1, "stream": false, - "stop": "\n", "frequency_penalty": 0.0 }' -``` \ No newline at end of file +``` + diff --git a/cloud-service-providers/azure/azureml/cli/azureml_files/deployment.yml b/cloud-service-providers/azure/azureml/cli/azureml_files/deployment.yml index 942beb4..19febd1 100644 --- a/cloud-service-providers/azure/azureml/cli/azureml_files/deployment.yml +++ b/cloud-service-providers/azure/azureml/cli/azureml_files/deployment.yml @@ -3,7 +3,7 @@ name: deployment_name_placeholder endpoint_name: endpoint_name_placeholder environment: name: image_name_placeholder-env - image: acr_registry_placeholder.azurecr.io/image_name_placeholder:v1 + image: acr_registry_placeholder.azurecr.io/image_name_placeholder:latest inference_config: liveness_route: path: /v1/health/ready diff --git a/cloud-service-providers/azure/azureml/cli/config.sh b/cloud-service-providers/azure/azureml/cli/config.sh index a5c6707..eb77dba 100755 --- a/cloud-service-providers/azure/azureml/cli/config.sh +++ b/cloud-service-providers/azure/azureml/cli/config.sh @@ -38,7 +38,7 @@ email_address="mreyesgomez@nvidia.com" acr_registry_name="nimdeploydemowsacr" image_name="llama3_8b_nim_ncd" -ngc_container="nvcr.io/nim/meta/llama3-8b-instruct:1.0.0" +ngc_container="nvcr.io/nim/meta/llama3-8b-instruct:1.0.3" # Endpoint related information endpoint_name="llama3-8b-nim-endpoint-aml-1" From ae5d76a1aa190095e58d245157b9a389c51ccd8c Mon Sep 17 00:00:00 2001 From: mreyesgomez Date: Sun, 20 Oct 2024 20:03:46 -0700 Subject: [PATCH 3/4] Fixing querying code --- cloud-service-providers/azure/azureml/cli/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud-service-providers/azure/azureml/cli/README.md b/cloud-service-providers/azure/azureml/cli/README.md index dd133fa..1d44b2e 100755 --- a/cloud-service-providers/azure/azureml/cli/README.md +++ b/cloud-service-providers/azure/azureml/cli/README.md @@ -89,7 +89,7 @@ Required commands are provided in the `5_create_deployment.sh` script. ./5_create_deployment.sh ``` -## Verify Your Connection +## Querying the model Verify your deployment using the `test_chat_completions.sh` script. Modify the URL to your endpoint URL and add the following headers: `-H 'Authorization: Bearer ' -H 'azureml-model-deployment: '` @@ -100,7 +100,7 @@ For example: curl -X 'POST' \ 'https://llama3-8b-nim-endpoint-aml-1.southcentralus.inference.ml.azure.com/v1/chat/completions' \ -H 'accept: application/json' \ - -H 'Authorization: Bearer hNMiUaTjQGYT1FobgtzOEaYJ1ei6lPYt' \ + -H 'Authorization: Bearer XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' \ -H 'azureml-model-deployment: llama3-8b-nim-deployment-aml-1' \ -H 'Content-Type: application/json' \ -d '{ From 6b390b964465eb9fb332603c94a768371251c1a0 Mon Sep 17 00:00:00 2001 From: mreyesgomez <47162567+mreyesgomez@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:32:45 -0700 Subject: [PATCH 4/4] Update config.sh --- .../azure/azureml/cli/config.sh | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cloud-service-providers/azure/azureml/cli/config.sh b/cloud-service-providers/azure/azureml/cli/config.sh index eb77dba..49cb9c6 100755 --- a/cloud-service-providers/azure/azureml/cli/config.sh +++ b/cloud-service-providers/azure/azureml/cli/config.sh @@ -22,23 +22,21 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # AzureML Workspace and corresponding container registry related information -subscription_id="ab221ca4-f098-422d-ab2f-5073b3851e68" -resource_group="NIMDeploDemo_rg" -workspace="NIMDeployDemo_ws" -location="southcentralus" +subscription_id="" +resource_group="" +workspace="" +location="" # eg: "southcentralus", "westeurope" etc. # Azure keyvault creation related information -ngc_api_key="azdwMG00YXNicWVzcnFrN3ZqMnFtcTAyb2U6NzRhZTUxODgtNzBhMS00ZDIzLThlODktZWNkNGQwMTE5OWRh" +ngc_api_key="" keyvault_name="NGC-Credentials" -email_address="mreyesgomez@nvidia.com" +email_address="" # Container related information # NOTE: Verify that your AML workspace can access this ACR - -acr_registry_name="nimdeploydemowsacr" - -image_name="llama3_8b_nim_ncd" -ngc_container="nvcr.io/nim/meta/llama3-8b-instruct:1.0.3" +acr_registry_name="" +image_name="" +ngc_container="nvcr.io/nim/meta/llama3-8b-instruct:1.0.0" # Endpoint related information endpoint_name="llama3-8b-nim-endpoint-aml-1"