From 42b62be28e41c15eb4ddb10f4fd633041d54b144 Mon Sep 17 00:00:00 2001 From: Sushil Kumar Date: Thu, 5 May 2022 13:41:10 -0700 Subject: [PATCH] Code updated to use acr images and updated script arguments --- deploy/README.md | 13 ++-- deploy/configure.sh | 2 +- .../infra/create_cpu_batch_account_pool.bicep | 11 ++++ deploy/use_pre_provisioned_batch_account.sh | 59 +++++-------------- 4 files changed, 30 insertions(+), 55 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index e58b3a5..d2239d6 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -346,12 +346,10 @@ To use the script, first deploy your infrastructure as usual and then use [use_p ./deploy/install.sh # Execute use_pre_provisioned_batch_account.sh script -./deploy/use_pre_provisioned_batch_account.sh \ - \ - \ - +./deploy/use_pre_provisioned_batch_account.sh ``` -refers to the name of the pipeline that is to be package for deployment to your Synapse Workspace. Allowed values are custom-vision-model, custom-vision-model-v2. +`pipelineName`: refers to the name of the pipeline that is to be package for deployment to your Synapse Workspace. Allowed values are `custom-vision-model`, `custom-vision-model-v2`(default). + For example: ```bash @@ -359,10 +357,7 @@ For example: ./deploy/install.sh aoi westus demo # Execute use_pre_provisioned_batch_account.sh script -./deploy/use_pre_provisioned_batch_account.sh aoi \ - existing-batch-account-name existing-batch-resource-group-name existing-batch-account-storage-account-name \ - raw-data-storage-account-name raw-data-storage-account-resource-group-name \ - custom-vision-model-v2 +./deploy/use_pre_provisioned_batch_account.sh aoi existing-batch-account-name custom-vision-model-v2 ``` **Note**: To use the script both the infrastructure(which includes syanpse workspace) and batch account should exist in **same location**. \ No newline at end of file diff --git a/deploy/configure.sh b/deploy/configure.sh index 795d236..6c760d8 100755 --- a/deploy/configure.sh +++ b/deploy/configure.sh @@ -34,7 +34,7 @@ if [[ -n $BATCH_ACCT ]] then az batch account login --name ${BATCH_ACCT} --resource-group ${ENVCODE}-orc-rg # create batch job for custom vision model - az batch job create --id ${ENVCODE}-data-cpu-pool --pool-id 'data-cpu-pool' --account-name ${BATCH_ACCT} --account-key ${BATCH_ACCT_KEY} + az batch job create --id ${ENVCODE}-data-cpu-pool --pool-id ${ENVCODE}-data-cpu-pool --account-name ${BATCH_ACCT} --account-key ${BATCH_ACCT_KEY} fi SYNAPSE_STORAGE_ACCT=$(az storage account list --query "[?tags.store && tags.store == 'synapse'].name" -o tsv -g $1-pipeline-rg) echo $SYNAPSE_STORAGE_ACCT diff --git a/deploy/infra/create_cpu_batch_account_pool.bicep b/deploy/infra/create_cpu_batch_account_pool.bicep index 83ae40b..4a1623e 100644 --- a/deploy/infra/create_cpu_batch_account_pool.bicep +++ b/deploy/infra/create_cpu_batch_account_pool.bicep @@ -15,6 +15,9 @@ param batchAccountPoolMountAccountName string param batchAccountPoolMountAccountKey string param batchAccountPoolMountFileUrl string +// ACR information +param acrName string = '' + // Parameters with default values for Data Fetch Batch Account Pool param batchAccountCpuOnlyPoolName string = 'data-cpu-pool' param batchAccountCpuOnlyPoolVmSize string = 'standard_d2s_v3' @@ -31,6 +34,10 @@ resource batchAccount 'Microsoft.Batch/batchAccounts@2021-06-01' existing = { scope: resourceGroup(batchAccountResourceGroup) } +resource containerRepository 'Microsoft.ContainerRegistry/registries@2021-12-01-preview' existing = if (acrName != '') { + name: acrName +} + module batchAccountPoolCheck 'modules/batch.account.pool.exists.bicep' = { name: '${namingPrefix}-batch-account-pool-exists' scope: resourceGroup(batchAccountResourceGroup) @@ -63,9 +70,13 @@ module batchAccountCpuOnlyPool 'modules/batch.account.pools.bicep' = { azureFileShareConfigurationAzureFileUrl: batchAccountPoolMountFileUrl azureFileShareConfigurationMountOptions: '-o vers=3.0,dir_mode=0777,file_mode=0777,sec=ntlmssp' azureFileShareConfigurationRelativeMountPath: 'S' + containerRegistryServer: containerRepository.properties.loginServer + containerRegistryUsername: containerRepository.listCredentials().username + containerRegistryPassword: containerRepository.listCredentials().passwords[0].value batchPoolExists: batchAccountPoolCheck.outputs.batchPoolExists } dependsOn: [ + containerRepository batchAccount batchAccountPoolCheck ] diff --git a/deploy/use_pre_provisioned_batch_account.sh b/deploy/use_pre_provisioned_batch_account.sh index 52bda41..ec50e83 100755 --- a/deploy/use_pre_provisioned_batch_account.sh +++ b/deploy/use_pre_provisioned_batch_account.sh @@ -6,20 +6,15 @@ PRJ_ROOT="$(cd `dirname "${BASH_SOURCE}"`/..; pwd)" environment_code=${1} - target_batch_account_name=${2} -target_batch_account_resource_group_name=${3} -target_batch_account_storage_account_name=${4} - -target_batch_pool_mount_storage_account_name=${5} -target_batch_pool_mount_storage_account_resource_group_name=${6} - -pipeline_name=${7} -target_batch_account_pool_name=${10:-"${environment_code}-data-cpu-pool"} -batch_account_role=${11:-"Contributor"} +pipeline_name=${3:-'custom-vision-model-v2'} +target_batch_account_role=${4:-"Contributor"} +target_batch_account_pool_name=${5:-"${environment_code}-data-cpu-pool"} +target_batch_account_storage_account_name=${6:-""} -set -e +target_batch_pool_mount_storage_account_name=${7:-""} +target_batch_pool_mount_storage_account_resource_group_name=${8:-""} if [[ -z "$environment_code" ]] then @@ -33,30 +28,13 @@ if [[ -z "$target_batch_account_name" ]] exit 1 fi -if [[ -z "$target_batch_account_resource_group_name" ]] - then - echo "Batch Account Resource Group Name value not supplied" - exit 1 -fi - -if [[ -z "$target_batch_account_storage_account_name" ]] - then - echo "Batch Account Storage Account Storage Account Name value not supplied" - exit 1 -fi - -if [[ -z "$target_batch_pool_mount_storage_account_name" ]] - then - echo "Storage Account Name For Mounting to Batch Pool not supplied" - exit 1 -fi - -if [[ -z "$target_batch_pool_mount_storage_account_resource_group_name" ]] - then - echo "Storage Account Resource Group Name For Mounting to Batch Pool not supplied" - exit 1 -fi +set -e +target_batch_account_id=$(az batch account list --query "[?name == '${target_batch_account_name}'].id" -o tsv) +target_batch_account_resource_group_name=$(az resource show --ids ${target_batch_account_id} --query resourceGroup -o tsv) +target_batch_account_storage_account_name=$(az storage account list --resource-group $target_batch_account_resource_group_name --query [0].name -o tsv) +target_batch_pool_mount_storage_account_resource_group_name="${environment_code}-data-rg" +target_batch_pool_mount_storage_account_name=$(az storage account list --resource-group $target_batch_pool_mount_storage_account_resource_group_name --query [0].name -o tsv) source_synapse_workspace_name="${environment_code}-pipeline-syn-ws" source_synapse_resource_group_name="${environment_code}-pipeline-rg" # Verify that infrastructure and batch account are in same location @@ -72,7 +50,7 @@ python3 ${PRJ_ROOT}/deploy/batch_account.py \ --env_code ${environment_code} \ --target_batch_account_name ${target_batch_account_name} \ --target_batch_account_resource_group_name ${target_batch_account_resource_group_name} \ - --batch_account_role ${batch_account_role} + --batch_account_role ${target_batch_account_role} # Deploy Batch Pool target_batch_pool_mount_storage_account_key=$(az storage account keys list --resource-group ${target_batch_pool_mount_storage_account_resource_group_name} --account-name ${target_batch_pool_mount_storage_account_name} --query "[0].value" --output tsv) @@ -85,6 +63,7 @@ az deployment group create --resource-group "${environment_code}-orc-rg" \ --parameters \ environmentCode=${environment_code} \ projectName="orc" \ + acrName="${environment_code}orcacr"\ batchAccountName=${target_batch_account_name} \ batchAccountResourceGroup=${target_batch_account_resource_group_name} \ batchAccountCpuOnlyPoolName="${target_batch_account_pool_name}" \ @@ -149,14 +128,4 @@ PACKAGING_SCRIPT="python3 ${PRJ_ROOT}/deploy/package.py --raw_storage_account_na --location $batch_account_location \ --pipeline_name $pipeline_name" -if [[ -z "$pipeline_name" ]] - then - echo "Pipeline Name value not supplied!!! So not packaging the Pipeline" - echo - echo "Please execute following command suffixed with pipeline to package the pipeline" - echo - echo $PACKAGING_SCRIPT - exit 1 -fi - $PACKAGING_SCRIPT \ No newline at end of file