fix pyarrow vulnerability in vision envs (#1806)

* fixing pyarrow vulnerabilities for dnn-vision-gpu * Added pyarrow-hotfix to fix vulnerabilities * Version up environments
Azure · Nov 23, 2023 · c63c304 · c63c304
1 parent b322af8
commit c63c304
Show file tree

Hide file tree

Showing 11 changed files with 14 additions and 17 deletions.
diff --git a/assets/training/finetune_acft_image/environments/acft_image_huggingface/asset.yaml b/assets/training/finetune_acft_image/environments/acft_image_huggingface/asset.yaml
@@ -1,5 +1,5 @@
 name: acft-transformers-image-gpu
-version: "8"
+version: "9"
 type: environment
 spec: spec.yaml
 extra_config: environment.yaml

diff --git a/...training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt b/...training/finetune_acft_image/environments/acft_image_huggingface/context/requirements.txt
@@ -6,10 +6,8 @@ azureml-acft-image-components=={{latest-pypi-version}}
 azureml-core==1.53.0
 azure-ai-ml==1.11.0
 requests
-datasets==2.14.5
+datasets==2.14.7
 transformers==4.34.1
-# Note: pinning transformers[sentencepiece], as sentencepiece is required by optimum lib, to avoid version conflict
-transformers[sentencepiece]==4.34.1
 accelerate==0.23.0
 optimum==1.13.1
 diffusers==0.20.2

diff --git a/assets/training/finetune_acft_image/environments/acft_image_mmdetection/asset.yaml b/assets/training/finetune_acft_image/environments/acft_image_mmdetection/asset.yaml
@@ -1,5 +1,5 @@
 name: acft-mmdetection-image-gpu
-version: "8"
+version: "9"
 type: environment
 spec: spec.yaml
 extra_config: environment.yaml

diff --git a/...training/finetune_acft_image/environments/acft_image_mmdetection/context/requirements.txt b/...training/finetune_acft_image/environments/acft_image_mmdetection/context/requirements.txt
@@ -6,10 +6,8 @@ azureml-acft-image-components=={{latest-pypi-version}}
 azureml-core==1.53.0
 azure-ai-ml==1.11.0
 requests
-datasets==2.14.5
+datasets==2.14.7
 transformers==4.34.1
-# Note: pinning transformers[sentencepiece], as sentencepiece is required by optimum lib, to avoid version conflict
-transformers[sentencepiece]==4.34.1
 accelerate==0.23.0
 # NOTE: optimum is pinned to old version because with new version, we see big drop in run time performance for MMD
 optimum==1.8.8

diff --git a/assets/training/finetune_acft_image/environments/acft_video_mmtracking/asset.yaml b/assets/training/finetune_acft_image/environments/acft_video_mmtracking/asset.yaml
@@ -1,5 +1,5 @@
 name: acft-mmtracking-video-gpu
-version: "3"
+version: "4"
 type: environment
 spec: spec.yaml
 extra_config: environment.yaml

diff --git a/.../training/finetune_acft_image/environments/acft_video_mmtracking/context/requirements.txt b/.../training/finetune_acft_image/environments/acft_video_mmtracking/context/requirements.txt
@@ -6,10 +6,8 @@ azureml-acft-image-components=={{latest-pypi-version}}
 azureml-core==1.53.0
 azure-ai-ml==1.11.0
 requests
-datasets==2.14.5
+datasets==2.14.7
 transformers==4.34.1
-# Note: pinning transformers[sentencepiece], as sentencepiece is required by optimum lib, to avoid version conflict
-transformers[sentencepiece]==4.34.1
 accelerate==0.23.0
 # NOTE: optimum is pinned to old version because with new version, we see big drop in run time performance for MMD
 optimum==1.8.8

diff --git a/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/asset.yaml b/assets/training/finetune_acft_image/environments/acpt_image_framework_selector/asset.yaml
@@ -1,5 +1,5 @@
 name: acpt-automl-image-framework-selector-gpu
-version: "8"
+version: "9"
 type: environment
 spec: spec.yaml
 extra_config: environment.yaml

diff --git a/...g/finetune_acft_image/environments/acpt_image_framework_selector/context/requirements.txt b/...g/finetune_acft_image/environments/acpt_image_framework_selector/context/requirements.txt
@@ -6,4 +6,5 @@ azureml-mlflow
 azureml-core
 requests
 certifi==2023.07.22
-setuptools==67.6.0
+setuptools==67.6.0
+pyarrow-hotfix
diff --git a/...odel_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt b/...odel_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-is-requirements.txt
@@ -1,6 +1,6 @@
 mlflow==2.6.0
 cloudpickle==2.2.1
-datasets==2.14.5
+datasets==2.14.7
 openmim==0.3.9
 torch==2.0.1
 torchvision==0.15.2

diff --git a/...odel_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt b/...odel_management/src/azureml/model/mgmt/processors/pyfunc/vision/mmdet-od-requirements.txt
@@ -1,6 +1,6 @@
 mlflow==2.6.0
 cloudpickle==2.2.1
-datasets==2.14.5
+datasets==2.14.7
 openmim==0.3.9
 torch==2.0.1
 torchvision==0.15.2

diff --git a/assets/training/vision/environments/automl-dnn-vision-gpu/context/conda_dependencies.yaml b/assets/training/vision/environments/automl-dnn-vision-gpu/context/conda_dependencies.yaml
@@ -15,6 +15,7 @@ dependencies:
 - wheel=0.38.1
 - pip:
   - cryptography>=41.0.4
+  - pyarrow-hotfix>=0.5
   - azureml-mlflow=={{latest-pypi-version}}
   - azureml-dataset-runtime=={{latest-pypi-version}}
   - azureml-telemetry=={{latest-pypi-version}}
@@ -26,4 +27,5 @@ dependencies:
   - azureml-interpret=={{latest-pypi-version}}
   - azureml-train-automl-runtime=={{latest-pypi-version}}
   - azureml-automl-dnn-vision=={{latest-pypi-version}}
-  - azureml-dataprep>=2.24.4
+  - azureml-dataprep>=2.24.4
+  - pyarrow-hotfix