From 9dad827b1d5232444855b206dd3f55c6c55d7ccf Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 11:24:53 -0700 Subject: [PATCH 1/7] env updates --- .../responsibleai-text/context/conda_dependencies.yaml | 2 +- .../environments/responsibleai-vision/context/Dockerfile | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml index f378fe0bf0..5148d23625 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml @@ -12,7 +12,7 @@ dependencies: - captum - cpuonly - pip: - - numpy==1.22.0 + - numpy==1.23.0 - ml-wrappers~=0.5.5 - responsibleai~=0.36.0 - raiwidgets~=0.36.0 diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index 83df4a347b..84f25d4943 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -31,6 +31,9 @@ RUN pip install 'azureml-core=={{latest-pypi-version}}' \ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ 'azureml-automl-dnn-vision=={{latest-pypi-version}}' +# Install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue +RUN conda install -p $AZUREML_CONDA_ENVIRONMENT_PATH torch>=2.2.2 + RUN pip install 'shap==0.41.0' \ 'scikit-learn>=1.5.1' \ 'interpret-community==0.31.0' \ From ac294e7df94c1bc3f0cf07bf8c4dfff03517f47d Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 11:42:58 -0700 Subject: [PATCH 2/7] pytorch channel fix --- .../environments/responsibleai-vision/context/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index 84f25d4943..da6c2ff58a 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -32,7 +32,7 @@ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ 'azureml-automl-dnn-vision=={{latest-pypi-version}}' # Install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue -RUN conda install -p $AZUREML_CONDA_ENVIRONMENT_PATH torch>=2.2.2 +RUN conda install -c pytorch -p $AZUREML_CONDA_ENVIRONMENT_PATH torch>=2.2.2 RUN pip install 'shap==0.41.0' \ 'scikit-learn>=1.5.1' \ From 1ba2db52e19cd795245185e09d00f76dc40380de Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 11:53:53 -0700 Subject: [PATCH 3/7] pytorch versio bump --- .../responsibleai-text/context/conda_dependencies.yaml | 2 +- .../responsibleai-vision/context/conda_dependencies.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml index 5148d23625..7aa2ed5d7b 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml @@ -8,7 +8,7 @@ channels: dependencies: - python=3.9 - pip - - pytorch + - pytorch>=2.2.2 - captum - cpuonly - pip: diff --git a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml index 538de63f11..247e9310e7 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml @@ -7,7 +7,7 @@ channels: dependencies: - python=3.9 - pip - - pytorch + - pytorch>=2.2.2 - torchvision - captum - cpuonly From 971b05391752cda1c227e531ea5689ce73ca2d88 Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 12:45:42 -0700 Subject: [PATCH 4/7] env updates --- .../environments/responsibleai-text/context/Dockerfile | 3 ++- .../responsibleai-text/context/conda_dependencies.yaml | 1 + .../environments/responsibleai-vision/context/Dockerfile | 4 ++-- .../responsibleai-vision/context/conda_dependencies.yaml | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/assets/responsibleai/environments/responsibleai-text/context/Dockerfile b/assets/responsibleai/environments/responsibleai-text/context/Dockerfile index 92a922fed5..bbe24c3851 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-text/context/Dockerfile @@ -7,7 +7,8 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Create conda environment COPY conda_dependencies.yaml . -RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml -q && \ +# TODO: revert back -q after env is debugged +RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml && \ rm conda_dependencies.yaml && \ conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip cache purge && \ conda clean -a -y diff --git a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml index 7aa2ed5d7b..d08c0e6a4e 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml @@ -26,3 +26,4 @@ dependencies: - datasets - typing-extensions<4.6.1 - gevent-websocket + - torch>=2.2.2 diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index da6c2ff58a..25f9a84e1e 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -31,8 +31,8 @@ RUN pip install 'azureml-core=={{latest-pypi-version}}' \ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ 'azureml-automl-dnn-vision=={{latest-pypi-version}}' -# Install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue -RUN conda install -c pytorch -p $AZUREML_CONDA_ENVIRONMENT_PATH torch>=2.2.2 +# Force install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue +RUN conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip install torch>=2.2.2 --force-reinstall --no-deps RUN pip install 'shap==0.41.0' \ 'scikit-learn>=1.5.1' \ diff --git a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml index 247e9310e7..2c5ff0de9c 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml @@ -26,3 +26,4 @@ dependencies: - ipython<8.13 - typing-extensions<4.6.1 - gevent-websocket + - torch>=2.2.2 From 3cb9a56382e001400eb11a685e6baa273a236f89 Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 12:57:58 -0700 Subject: [PATCH 5/7] env updates --- .../responsibleai-text/context/conda_dependencies.yaml | 2 +- .../environments/responsibleai-vision/context/Dockerfile | 6 ++++-- .../responsibleai-vision/context/conda_dependencies.yaml | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml index d08c0e6a4e..be898ff726 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-text/context/conda_dependencies.yaml @@ -24,6 +24,6 @@ dependencies: - transformers>=4.17.0 - nlp-feature-extractors==0.1.0 - datasets - - typing-extensions<4.6.1 + - typing-extensions>=4.8.0 - gevent-websocket - torch>=2.2.2 diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index 25f9a84e1e..ea2b165976 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -31,8 +31,10 @@ RUN pip install 'azureml-core=={{latest-pypi-version}}' \ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ 'azureml-automl-dnn-vision=={{latest-pypi-version}}' -# Force install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue -RUN conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip install torch>=2.2.2 --force-reinstall --no-deps +# # Force install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue +# RUN conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip install torch>=2.2.2 --force-reinstall --no-deps +# Install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue +RUN conda install -c pytorch -p $AZUREML_CONDA_ENVIRONMENT_PATH pytorch>=2.2.2 RUN pip install 'shap==0.41.0' \ 'scikit-learn>=1.5.1' \ diff --git a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml index 2c5ff0de9c..fefb0e0748 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml @@ -24,6 +24,6 @@ dependencies: - mltable==1.5.0 - fastai - ipython<8.13 - - typing-extensions<4.6.1 + - typing-extensions>=4.8.0 - gevent-websocket - - torch>=2.2.2 + # - torch>=2.2.2 From f47534f75daf1b077d01af41c90cb7ae19f9a952 Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 13:25:43 -0700 Subject: [PATCH 6/7] env updates & ckpt --- .../environments/responsibleai-text/context/Dockerfile | 3 +-- .../environments/responsibleai-vision/context/Dockerfile | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/responsibleai/environments/responsibleai-text/context/Dockerfile b/assets/responsibleai/environments/responsibleai-text/context/Dockerfile index bbe24c3851..92a922fed5 100644 --- a/assets/responsibleai/environments/responsibleai-text/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-text/context/Dockerfile @@ -7,8 +7,7 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Create conda environment COPY conda_dependencies.yaml . -# TODO: revert back -q after env is debugged -RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml && \ +RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml -q && \ rm conda_dependencies.yaml && \ conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip cache purge && \ conda clean -a -y diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index ea2b165976..6092cee069 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -34,6 +34,7 @@ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ # # Force install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue # RUN conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip install torch>=2.2.2 --force-reinstall --no-deps # Install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue +RUN conda remove pytorch -y RUN conda install -c pytorch -p $AZUREML_CONDA_ENVIRONMENT_PATH pytorch>=2.2.2 RUN pip install 'shap==0.41.0' \ From 9b51caa913208272bde7265bbb3547cd16705a88 Mon Sep 17 00:00:00 2001 From: Advitya Gemawat Date: Thu, 25 Jul 2024 13:43:39 -0700 Subject: [PATCH 7/7] torch updates for vision image --- .../environments/responsibleai-vision/context/Dockerfile | 9 +++------ .../responsibleai-vision/context/conda_dependencies.yaml | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile index 6092cee069..07c9b09da3 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile +++ b/assets/responsibleai/environments/responsibleai-vision/context/Dockerfile @@ -31,12 +31,6 @@ RUN pip install 'azureml-core=={{latest-pypi-version}}' \ RUN pip install 'azureml-dataset-runtime=={{latest-pypi-version}}' \ 'azureml-automl-dnn-vision=={{latest-pypi-version}}' -# # Force install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue -# RUN conda run -p $AZUREML_CONDA_ENVIRONMENT_PATH pip install torch>=2.2.2 --force-reinstall --no-deps -# Install torch version >= 2.2.2 to override azureml-automl-dnn-vision version and fix remote execution issue -RUN conda remove pytorch -y -RUN conda install -c pytorch -p $AZUREML_CONDA_ENVIRONMENT_PATH pytorch>=2.2.2 - RUN pip install 'shap==0.41.0' \ 'scikit-learn>=1.5.1' \ 'interpret-community==0.31.0' \ @@ -59,6 +53,9 @@ RUN pip install 'Werkzeug>=3.0.3' \ 'tqdm>=4.66.3' \ 'onnx>=1.16.0' +# To resolve CVE-2024-5480 vulnerability issue for torch < 2.2.2 +RUN pip install 'torch>=2.2.2' + RUN pip freeze # This is needed for mpi to locate libpython diff --git a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml index fefb0e0748..52e81f790f 100644 --- a/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml +++ b/assets/responsibleai/environments/responsibleai-vision/context/conda_dependencies.yaml @@ -26,4 +26,4 @@ dependencies: - ipython<8.13 - typing-extensions>=4.8.0 - gevent-websocket - # - torch>=2.2.2 + - torch>=2.2.2