Merge branch 'main' into dependabot/pip/docker/py/renku-2.7.0

SwissDataScienceCenter · Oct 17, 2023 · dafe098 · dafe098
2 parents 28e767f + 02d46d5
commit dafe098
Show file tree

Hide file tree

Showing 2 changed files with 92 additions and 18 deletions.
diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml
@@ -71,7 +71,7 @@ jobs:
     - name: Set up QEMU
       uses: docker/[email protected]
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - name: Docker meta
       id: meta
       uses: docker/metadata-action@v4
@@ -80,7 +80,7 @@ jobs:
         tags: |
           type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}-
     - name: Build and load
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}      
@@ -103,7 +103,7 @@ jobs:
     # export the default base image for other jobs to use
     - name: Build and export
       if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
             BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}
@@ -114,6 +114,72 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         provenance: false
 
+  build-py-homedir-env:
+    needs: lint
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        BASE_PYTHON_VERSION:
+          - "3.10"
+          - "3.9"
+          - "3.8"
+    steps:
+    - name: Docker Login
+      uses: Azure/docker-login@v1
+      with:
+        username: ${{ secrets.RENKU_DOCKER_USERNAME }}
+        password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
+    - uses: actions/checkout@v3
+    - name: Set up QEMU
+      uses: docker/[email protected]
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v4
+      with:
+        images: ${{ env.DOCKER_PREFIX }}-py-homedir-envs
+        tags: |
+          type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}-
+    - name: Build and load
+      uses: docker/build-push-action@v5
+      with:
+        build-args: |
+          BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}      
+          CONDA_ENVS_DIRS=/home/jovyan/work/envs
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        context: docker/py
+        labels: ${{ steps.meta.outputs.labels }}
+        load: true
+        tags: ${{ steps.meta.outputs.tags }}
+        provenance: false
+    - name: Image Acceptance Tests
+      uses: cypress-io/github-action@v4
+      env:
+        TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-py-homedir-envs:${{ steps.meta.outputs.version }}
+        TEST_USER_NAME: jovyan
+        TEST_SPEC: jupyterlab.cy.ts
+      with:
+        working-directory: tests
+        command: npx mocha -r ts-node/register index.ts
+    # export the default base image for other jobs to use
+    - name: Build and export
+      if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION
+      uses: docker/build-push-action@v5
+      with:
+        build-args: |
+            BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}
+            CONDA_ENVS_DIRS=/home/jovyan/work/envs
+        cache-from: type=gha
+        context: docker/py
+        labels: ${{ steps.meta.outputs.labels }}
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        provenance: false
+
   build-py-multiarch:
     needs: build-py
     runs-on: ubuntu-latest
@@ -135,7 +201,7 @@ jobs:
     - name: Set up QEMU
       uses: docker/[email protected]
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - name: Docker meta
       id: meta
       uses: docker/metadata-action@v4
@@ -146,7 +212,7 @@ jobs:
           type=semver,pattern={{version}},prefix=${{ matrix.BASE_PYTHON_VERSION }}-
           type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION }} 
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: docker/py
         push: true
@@ -213,7 +279,7 @@ jobs:
         echo "sha_short=$sha_short" >> $GITHUB_OUTPUT
         echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - name: Docker meta
       id: meta
       uses: docker/metadata-action@v4
@@ -224,7 +290,7 @@ jobs:
           type=semver,pattern={{version}},prefix=${{ matrix.CUDA_VERSION }}-
           type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.CUDA_VERSION == '11.7' }}
     - name: Build and load
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}
@@ -252,7 +318,7 @@ jobs:
         working-directory: tests
         command: npx mocha -r ts-node/register index.ts
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}
@@ -295,7 +361,7 @@ jobs:
     - name: Set up QEMU
       uses: docker/[email protected]
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - name: Docker meta
       id: meta
       uses: docker/metadata-action@v4
@@ -306,7 +372,7 @@ jobs:
           type=semver,pattern={{version}},prefix=
           type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }},prefix=
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}
@@ -333,7 +399,7 @@ jobs:
         username: ${{ secrets.RENKU_DOCKER_USERNAME }}
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - uses: actions/checkout@v3
     - name: Set outputs
       id: vars
@@ -351,7 +417,7 @@ jobs:
           type=semver,pattern={{version}},prefix=
           type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }},prefix=
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           BASE_IMAGE=${{ env.DOCKER_PREFIX }}-vnc:${{ steps.vars.outputs.sha_short }}
@@ -388,7 +454,7 @@ jobs:
     - name: Set up QEMU
       uses: docker/[email protected]
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - name: Set outputs
       id: vars
       run: |
@@ -405,7 +471,7 @@ jobs:
           type=semver,pattern={{version}},prefix=${{ matrix.JULIA_VERSION }}-
           type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.JULIA_VERSION == '1.9.0' }}
     - name: Build and load
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}
@@ -428,7 +494,7 @@ jobs:
         working-directory: tests
         command: npx mocha -r ts-node/register index.ts
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}
@@ -487,7 +553,7 @@ jobs:
 
     - uses: actions/checkout@v3
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2.0.0
+      uses: docker/setup-buildx-action@v3
     - name: Set outputs
       id: vars
       run: |
@@ -503,8 +569,13 @@ jobs:
           type=sha,prefix=${{ matrix.RVERSION }}-
           type=semver,pattern={{version}},prefix=${{ matrix.RVERSION }}-
           type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.RVERSION == 'devel' }}
+    - name: Clean cache
+      run: |
+        df -h
+        rm -rf /opt/hostedtoolcache/*
+        df -h
     - name: Build and load
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}
@@ -525,9 +596,10 @@ jobs:
         TEST_SPEC: rstudio.cy.ts
       with:
         working-directory: tests
+        start: docker images
         command: npx mocha -r ts-node/register index.ts
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         build-args: |
           RENKU_BASE=${{ steps.vars.outputs.renku_base }}

diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile
@@ -105,3 +105,5 @@ COPY --chown=1000:100 --from=builder /opt/conda /opt/conda
 COPY --chown=1000:100 --from=builder "$HOME/.renku" "$HOME/.renku"
 RUN ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku" && \
     ln -s "$HOME/.renku/venv/bin/_toil_worker" "$HOME/.renku/bin/"
+ARG CONDA_ENVS_DIRS=/opt/conda/envs
+ENV CONDA_ENVS_DIRS=${CONDA_ENVS_DIRS}