remove git-lfs=2.12.0 requirement

use mamba instead of conda when possible remove libc symlink hack use apt instead of apt-get
SwissDataScienceCenter · Dec 15, 2021 · 0ae1458 · 0ae1458
1 parent d084850
commit 0ae1458
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 48 deletions.
diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml
@@ -5,11 +5,21 @@ on:
     types:
     - synchronize
 
+# NOTE: although docker platform specs are in the form of <OS>/<ARCH>
+# since these variables are used for other purposes as well, we are
+# using the <OS>_<ARCH> form and will use a char-replace for the docker
+# format
 env:
-  DOCKER_PLATFORMS: linux/amd64,linux/arm64
+  DOCKER_PLATFORMS: linux_amd64,linux_arm64
+  BUILDX_CACHE: /tmp/docker-cache
 
 jobs:
   build-base-python-image:
+    strategy:
+      matrix:
+        include:
+          - platform: linux_amd64
+          - platform: linux_arm64
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -21,13 +31,14 @@ jobs:
         uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
       - name: Build Image
         run: |
-          docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/
-          docker save py > /tmp/py.tar
+          PLATFORM=`echo ${{matrix.platform}} | tr "_" /`
+          docker buildx build --cache-to=type=local,dest=${{env.BUILDX_CACHE}}/$PLATFORM --platform $PLATFORM -t py docker/py/
+          tar cf /tmp/py_${{matrix.platform}}.tar -C ${{env.BUILDX_CACHE}} $PLATFORM
       - name: Upload Image Artifact
         uses: actions/upload-artifact@v2
         with:
-          name: py
-          path: /tmp/py.tar
+          name: py_${{matrix.platform}}
+          path: /tmp/py_${{matrix.platform}}.tar
 
   acceptance-tests:
     needs: build-base-python-image
@@ -39,6 +50,7 @@ jobs:
           - image_name: cuda-tf
             build_arg: BASE_IMAGE=py
             test_user: jovyan
+            platform: linux/amd64
           - image_name: generic
             build_arg: RENKU_BASE=py
             test_user: jovyan
@@ -51,23 +63,54 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Download Base Image
+      - name: Download Base Image linux/amd64
+        uses: actions/download-artifact@v2
+        with:
+          name: py_linux_amd64
+          path: /tmp
+      - name: Download Base Image linux/arm64
         uses: actions/download-artifact@v2
         with:
-          name: py
+          name: py_linux_arm64
           path: /tmp
       - name: Set up QEMU
         # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
         uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
-      - name: Load Base Image in Docker
+      - name: Extract Docker caches
         run: |
-          docker load --input /tmp/py.tar
+          mkdir ${{env.BUILDX_CACHE}}
+
+          PLATFORMS="${{env.DOCKER_PLATFORMS}}"
+          IFS=,
+          for p in $PLATFORMS;
+          do
+            tar xf /tmp/py_${p}.tar -C ${{env.BUILDX_CACHE}}
+          done
       - name: Build images
         if: ${{ matrix.image_name != 'py' }}
         run: |
-          docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/
+          if [ "$DOCKER_PLATFORM" != "" ]; then
+            PLATFORMS=${DOCKER_PLATFORM}
+          else
+            PLATFORMS=`echo ${{env.DOCKER_PLATFORMS}} | tr "_" / | tr "," " "`
+          fi
+
+          CACHE_FROM=""
+          for p in $PLATFORMS;
+          do
+            CACHE_FROM+="--cache-from=type=local,src=${{env.BUILDX_CACHE}}/${p} "
+          done
+
+          docker buildx build \
+            $CACHE_FROM \
+            --platform ${PLATFORMS} \
+            -t ${{ matrix.image_name }} \
+            --build-arg ${{ matrix.build_arg }} \
+            docker/${{ matrix.image_name }}/
+        env:
+          DOCKER_PLATFORM: ${{ matrix.platform }}
       - name: Py Image Acceptance Tests
         uses: cypress-io/github-action@v2
         env:

diff --git a/.github/workflows/build_and_push_to_docker.yml b/.github/workflows/build_and_push_to_docker.yml
@@ -8,6 +8,11 @@ env:
 jobs:
 
   build-py:
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+          - platform: linux/arm64
 
     runs-on: ubuntu-latest
 
@@ -32,8 +37,7 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/py --tag $DOCKER_NAME-py:3.9-$LABEL
-        docker push $DOCKER_NAME-py:3.9-$LABEL
+        docker buildx build --platform ${{matrix.platform}} docker/py --tag $DOCKER_NAME-py:3.9-$LABEL --push
 
         # on master push latest image
         if [ "$REF" == "refs/heads/master" ]
@@ -54,6 +58,8 @@ jobs:
         EXTENSIONS:
           - cuda-tf
           - vnc
+        PLATFORMS:
+          - linux/amd64
 
     steps:
     - name: Docker Login
@@ -76,10 +82,10 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/$EXTENSION \
+        docker buildx build --platform ${{matrix.PLATFORMS}} docker/$EXTENSION \
           --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \
-          --tag $DOCKER_NAME-$EXTENSION:$LABEL
-        docker push $DOCKER_NAME-$EXTENSION:$LABEL
+          --tag $DOCKER_NAME-$EXTENSION:$LABEL \
+          --push
       env:
         EXTENSION: ${{ matrix.EXTENSIONS }}
         DOCKER_NAME: "renku/renkulab"
@@ -112,8 +118,8 @@ jobs:
         docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/batch \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE="python:3.9-slim-buster" \
-          --tag $DOCKER_NAME-batch:$LABEL
-        docker push $DOCKER_NAME-batch:$LABEL
+          --tag $DOCKER_NAME-batch:$LABEL \
+          --push
       env:
         DOCKER_NAME: "renku/renkulab"
 
@@ -151,8 +157,8 @@ jobs:
         export DOCKER_TAG="$JULIAVERSION-$LABEL"
         docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/julia \
           --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \
-          --tag $DOCKER_NAME-julia:$DOCKER_TAG
-        docker push $DOCKER_NAME-julia:$DOCKER_TAG
+          --tag $DOCKER_NAME-julia:$DOCKER_TAG \
+          --push
       env:
         DOCKER_NAME: "renku/renkulab"
         JULIAVERSION: ${{ matrix.JULIAVERSIONS }}
@@ -195,8 +201,8 @@ jobs:
         docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE="rocker/verse:${RVERSION}" \
-          --tag $DOCKER_NAME-r:$DOCKER_TAG
-        docker push $DOCKER_NAME-r:$DOCKER_TAG
+          --tag $DOCKER_NAME-r:$DOCKER_TAG \
+          --push
 
         # on master push latest image
         if [ "$REF" == "refs/heads/master" ] && [ "$RVERSION" == "4.0.5" ]
@@ -247,8 +253,8 @@ jobs:
         docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE=bioconductor/bioconductor_docker:${RELEASE} \
-          --tag $DOCKER_NAME-bioc:$DOCKER_TAG
-        docker push $DOCKER_NAME-bioc:$DOCKER_TAG
+          --tag $DOCKER_NAME-bioc:$DOCKER_TAG \
+          --push
 
         # on master push latest image
         if [ "$REF" == "refs/heads/master" ] && [ "$RELEASE" == "devel" ]

diff --git a/docker/batch/Dockerfile b/docker/batch/Dockerfile
@@ -11,10 +11,7 @@ LABEL maintainer="Swiss Data Science Center <[email protected]>"
 
 # install dependencies
 RUN apt-get update -y && \
-    apt-get install -y curl && \
-    curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-    apt-get update -y && \
-    apt-get install -y gcc git git-lfs nodejs && \
+    apt-get install -y curl gcc git-lfs nodejs && \
     apt-get purge && \
     apt-get clean && \
     apt autoremove --yes && \

diff --git a/docker/generic/Dockerfile b/docker/generic/Dockerfile
@@ -26,9 +26,7 @@ RUN apt-get update --fix-missing && \
     bzip2 \
     ca-certificates \
     curl \
-    gpg-agent && \
-    curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-    apt-get install -yq --no-install-recommends \
+    gpg-agent \
     git-lfs \
     gnupg \
     libglib2.0-0 \

diff --git a/docker/julia/Dockerfile b/docker/julia/Dockerfile
@@ -12,13 +12,19 @@ ENV JULIA_PKGDIR=/opt/julia
 # If this changes, you also need to change the checksum below
 ENV JULIA_VERSION=1.6.1
 
-RUN mkdir /opt/julia-${JULIA_VERSION} && \
-    cd /tmp && \
-    wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
-    echo "7c888adec3ea42afbfed2ce756ce1164a570d50fa7506c3f2e1e2cbc49d52506 *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
-    tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
-    rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
-RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
+RUN set -x && \
+    julia_arch=$(uname -m) && \
+    julia_short_arch="${julia_arch}" && \
+    if [ "${julia_short_arch}" == "x86_64" ]; then \
+      julia_short_arch="x64"; \
+    fi; \
+    julia_installer="julia-${JULIA_VERSION}-linux-${julia_arch}.tar.gz" && \
+    julia_major_minor=$(echo "${JULIA_VERSION}" | cut -d. -f 1,2) && \
+    mkdir "/opt/julia-${JULIA_VERSION}" && \
+    wget -q "https://julialang-s3.julialang.org/bin/linux/${julia_short_arch}/${julia_major_minor}/${julia_installer}" && \
+    tar xzf "${julia_installer}" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \
+    rm "${julia_installer}" && \
+    ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
 
 # Show Julia where conda libraries are \
 RUN mkdir /etc/julia && \

diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile
@@ -8,13 +8,11 @@ LABEL maintainer="Swiss Data Science Center <[email protected]>"
 USER root
 
 # Install additional dependencies and nice-to-have packages
-RUN apt-get update && apt-get install -yq --no-install-recommends \
+RUN apt update && apt install -yq --no-install-recommends \
     build-essential \
-    curl && \
-    curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
-    apt-get update && apt-get install -yq --no-install-recommends \
+    curl \
     git \
-    git-lfs=2.12.0 \
+    git-lfs \
     gnupg \
     graphviz \
     jq \
@@ -31,9 +29,8 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
     python-dev \
     unzip \
     vim && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Add a new group with id 1000 and arbitrary name
 # Note: this is to standardize groups between the R and jupyter images
@@ -52,10 +49,10 @@ RUN python3 -m pip install --no-cache -U pip && \
     rm -rf /home/${NB_USER}/.cache
 
 # fix https://github.com/SwissDataScienceCenter/renku-jupyter/issues/14
-RUN conda install gxx_linux-64 && \
+RUN mamba install -y gxx_linux-64 && \
     # jupyter sets channel priority to strict which often causes very long error messages
     conda config --system --set channel_priority flexible && \
-    conda clean --all -f -y
+    mamba clean --all -f -y
 
 # install renku-python
 ENV RENKU_DISABLE_VERSION_CHECK 1

diff --git a/docker/r/Dockerfile b/docker/r/Dockerfile
@@ -47,7 +47,6 @@ RUN apt-get update --fix-missing && \
     curl \
     gpg-agent && \
     curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
-    curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
     apt-get update && \
     apt-get install -yq --no-install-recommends \
     git-lfs \