From c63c3045c7d59a3674109e35bfe83cd3c17278ea Mon Sep 17 00:00:00 2001 From: nvijayrania <107195344+nvijayrania@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:46:53 +0530 Subject: [PATCH] fix pyarrow vulnerability in vision envs (#1806) * fixing pyarrow vulnerabilities for dnn-vision-gpu * Added pyarrow-hotfix to fix vulnerabilities * Version up environments --- .../environments/acft_image_huggingface/asset.yaml | 2 +- .../acft_image_huggingface/context/requirements.txt | 4 +--- .../environments/acft_image_mmdetection/asset.yaml | 2 +- .../acft_image_mmdetection/context/requirements.txt | 4 +--- .../environments/acft_video_mmtracking/asset.yaml | 2 +- .../acft_video_mmtracking/context/requirements.txt | 4 +--- .../environments/acpt_image_framework_selector/asset.yaml | 2 +- .../acpt_image_framework_selector/context/requirements.txt | 3 ++- .../mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt | 2 +- .../mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt | 2 +- .../automl-dnn-vision-gpu/context/conda_dependencies.yaml | 4 +++- 11 files changed, 14 insertions(+), 17 deletions(-) diff --git a/assets/training/finetune_acft_image/environments/acft_image_huggingface/asset.yaml b/assets/training/finetune_acft_image/environments/acft_image_huggingface/asset.yaml index 18cf4b5df3..796292561d 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_huggingface/asset.yaml +++ b/assets/training/finetune_acft_image/environments/acft_image_huggingface/asset.yaml @@ -1,5 +1,5 @@ name: acft-transformers-image-gpu -version: "8" +version: "9" type: environment spec: spec.yaml extra_config: environment.yaml diff --git a/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt b/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt index 37a2ea1481..923dfc4d96 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt +++ b/assets/training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt @@ -6,10 +6,8 @@ azureml-acft-image-components=={{latest-pypi-version}} azureml-core==1.53.0 azure-ai-ml==1.11.0 requests -datasets==2.14.5 +datasets==2.14.7 transformers==4.34.1 -# Note: pinning transformers[sentencepiece], as sentencepiece is required by optimum lib, to avoid version conflict -transformers[sentencepiece]==4.34.1 accelerate==0.23.0 optimum==1.13.1 diffusers==0.20.2 diff --git a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/asset.yaml b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/asset.yaml index 4e818c061e..cf1793011c 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/asset.yaml +++ b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/asset.yaml @@ -1,5 +1,5 @@ name: acft-mmdetection-image-gpu -version: "8" +version: "9" type: environment spec: spec.yaml extra_config: environment.yaml diff --git a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/requirements.txt b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/requirements.txt index 2853b09db8..8ce351b785 100644 --- a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/requirements.txt +++ b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/context/requirements.txt @@ -6,10 +6,8 @@ azureml-acft-image-components=={{latest-pypi-version}} azureml-core==1.53.0 azure-ai-ml==1.11.0 requests -datasets==2.14.5 +datasets==2.14.7 transformers==4.34.1 -# Note: pinning transformers[sentencepiece], as sentencepiece is required by optimum lib, to avoid version conflict -transformers[sentencepiece]==4.34.1 accelerate==0.23.0 # NOTE: optimum is pinned to old version because with new version, we see big drop in run time performance for MMD optimum==1.8.8 diff --git a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/asset.yaml b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/asset.yaml index abcf4443c4..b437ab8664 100644 --- a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/asset.yaml +++ b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/asset.yaml @@ -1,5 +1,5 @@ name: acft-mmtracking-video-gpu -version: "3" +version: "4" type: environment spec: spec.yaml extra_config: environment.yaml diff --git a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/requirements.txt b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/requirements.txt index 08e6a9d5c3..05adfac203 100644 --- a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/requirements.txt +++ b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/context/requirements.txt @@ -6,10 +6,8 @@ azureml-acft-image-components=={{latest-pypi-version}} azureml-core==1.53.0 azure-ai-ml==1.11.0 requests -datasets==2.14.5 +datasets==2.14.7 transformers==4.34.1 -# Note: pinning transformers[sentencepiece], as sentencepiece is required by optimum lib, to avoid version conflict -transformers[sentencepiece]==4.34.1 accelerate==0.23.0 # NOTE: optimum is pinned to old version because with new version, we see big drop in run time performance for MMD optimum==1.8.8 diff --git a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/asset.yaml b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/asset.yaml index 30d90d7a92..88eb903557 100644 --- a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/asset.yaml +++ b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/asset.yaml @@ -1,5 +1,5 @@ name: acpt-automl-image-framework-selector-gpu -version: "8" +version: "9" type: environment spec: spec.yaml extra_config: environment.yaml diff --git a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/requirements.txt b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/requirements.txt index f7cb63f23e..c57f40bd4d 100644 --- a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/requirements.txt +++ b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/context/requirements.txt @@ -6,4 +6,5 @@ azureml-mlflow azureml-core requests certifi==2023.07.22 -setuptools==67.6.0 \ No newline at end of file +setuptools==67.6.0 +pyarrow-hotfix \ No newline at end of file diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt index d282de91e8..e784dda2f6 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt @@ -1,6 +1,6 @@ mlflow==2.6.0 cloudpickle==2.2.1 -datasets==2.14.5 +datasets==2.14.7 openmim==0.3.9 torch==2.0.1 torchvision==0.15.2 diff --git a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt index 5e0c643564..00c43b9984 100644 --- a/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt +++ b/assets/training/model_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt @@ -1,6 +1,6 @@ mlflow==2.6.0 cloudpickle==2.2.1 -datasets==2.14.5 +datasets==2.14.7 openmim==0.3.9 torch==2.0.1 torchvision==0.15.2 diff --git a/assets/training/vision/environments/automl-dnn-vision-gpu/context/conda_dependencies.yaml b/assets/training/vision/environments/automl-dnn-vision-gpu/context/conda_dependencies.yaml index ae06d9aa3c..ae7af7765e 100644 --- a/assets/training/vision/environments/automl-dnn-vision-gpu/context/conda_dependencies.yaml +++ b/assets/training/vision/environments/automl-dnn-vision-gpu/context/conda_dependencies.yaml @@ -15,6 +15,7 @@ dependencies: - wheel=0.38.1 - pip: - cryptography>=41.0.4 + - pyarrow-hotfix>=0.5 - azureml-mlflow=={{latest-pypi-version}} - azureml-dataset-runtime=={{latest-pypi-version}} - azureml-telemetry=={{latest-pypi-version}} @@ -26,4 +27,5 @@ dependencies: - azureml-interpret=={{latest-pypi-version}} - azureml-train-automl-runtime=={{latest-pypi-version}} - azureml-automl-dnn-vision=={{latest-pypi-version}} - - azureml-dataprep>=2.24.4 \ No newline at end of file + - azureml-dataprep>=2.24.4 + - pyarrow-hotfix \ No newline at end of file