support multi-platform build using buildx

SwissDataScienceCenter · Dec 15, 2021 · d084850 · d084850
1 parent 0b41af7
commit d084850
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 16 deletions.
diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml
@@ -5,14 +5,23 @@ on:
     types:
     - synchronize
 
+env:
+  DOCKER_PLATFORMS: linux/amd64,linux/arm64
+
 jobs:
   build-base-python-image:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
       - name: Build Image
         run: |
-          docker build -t py docker/py/
+          docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/
           docker save py > /tmp/py.tar
       - name: Upload Image Artifact
         uses: actions/upload-artifact@v2
@@ -47,13 +56,18 @@ jobs:
         with:
           name: py
           path: /tmp
+      - name: Set up QEMU
+        # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
       - name: Load Base Image in Docker
         run: |
           docker load --input /tmp/py.tar
       - name: Build images
         if: ${{ matrix.image_name != 'py' }}
         run: |
-          docker build -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/
+          docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/
       - name: Py Image Acceptance Tests
         uses: cypress-io/github-action@v2
         env:

diff --git a/.github/workflows/build_and_push_to_docker.yml b/.github/workflows/build_and_push_to_docker.yml
@@ -2,6 +2,9 @@ name: Renku Docker Image CI
 
 on: [push]
 
+env:
+  DOCKER_PLATFORMS: linux/amd64,linux/arm64
+
 jobs:
 
   build-py:
@@ -16,6 +19,11 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project python-based docker images
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -24,7 +32,7 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker build docker/py --tag $DOCKER_NAME-py:3.9-$LABEL
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/py --tag $DOCKER_NAME-py:3.9-$LABEL
         docker push $DOCKER_NAME-py:3.9-$LABEL
 
         # on master push latest image
@@ -55,6 +63,11 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project python-based docker image extensions
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -63,7 +76,7 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker build docker/$EXTENSION \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/$EXTENSION \
           --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \
           --tag $DOCKER_NAME-$EXTENSION:$LABEL
         docker push $DOCKER_NAME-$EXTENSION:$LABEL
@@ -83,6 +96,11 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project python-based docker image for batch execution
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -91,7 +109,7 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker build docker/batch \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/batch \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE="python:3.9-slim-buster" \
           --tag $DOCKER_NAME-batch:$LABEL
@@ -117,6 +135,11 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project julia docker image extensions
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -126,7 +149,7 @@ jobs:
         fi
 
         export DOCKER_TAG="$JULIAVERSION-$LABEL"
-        docker build docker/julia \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/julia \
           --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \
           --tag $DOCKER_NAME-julia:$DOCKER_TAG
         docker push $DOCKER_NAME-julia:$DOCKER_TAG
@@ -155,6 +178,11 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project rocker docker images
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -164,7 +192,7 @@ jobs:
         fi
 
         export DOCKER_TAG="$RVERSION-$LABEL"
-        docker build docker/r \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE="rocker/verse:${RVERSION}" \
           --tag $DOCKER_NAME-r:$DOCKER_TAG
@@ -202,6 +230,11 @@ jobs:
         password: ${{ secrets.DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project bioconductor docker images
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -211,7 +244,7 @@ jobs:
         fi
 
         export DOCKER_TAG="$RELEASE-$LABEL"
-        docker build docker/r \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE=bioconductor/bioconductor_docker:${RELEASE} \
           --tag $DOCKER_NAME-bioc:$DOCKER_TAG

diff --git a/Makefile b/Makefile
@@ -26,6 +26,7 @@ extensions = \
 
 DOCKER_PREFIX?=renku/renkulab
 DOCKER_LABEL?=latest
+PLATFORMS?=linux/amd64,linux/arm64
 GIT_MASTER_HEAD_SHA:=$(shell git rev-parse --short=7 --verify HEAD)
 
 # for the empty version case:
@@ -66,14 +67,14 @@ pull:
 	done
 
 r: py
-	docker build docker/r \
+	docker buildx build --platform $(PLATFORMS) docker/r \
 		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		--build-arg RVERSION=$(RVERSION) \
 		-t $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) && \
 	docker tag $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) $(DOCKER_PREFIX)-r:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)$(R_TAG)
 
 bioc: py
-	docker build docker/bioc \
+	docker buildx build --platform $(PLATFORMS) docker/bioc \
 		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		--build-arg RELEASE=$(BIOC_VERSION) \
 		-t $(DOCKER_PREFIX)-bioc:$(DOCKER_LABEL)$(RENKU_TAG)$(BIOC_TAG) && \
@@ -82,26 +83,26 @@ bioc: py
 
 py:
 	cd docker/$@ && \
-	docker build \
+	docker buildx build --platform $(PLATFORMS) \
 		-t $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) . && \
 	docker tag $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-$@:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)
 
 cuda: py
-	docker build docker/cuda-tf \
+	docker buildx build --platform $(PLATFORMS) docker/cuda-tf \
 		--build-arg RENKU_PIP_SPEC=$(RENKU_PIP_SPEC) \
 		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA) \
 		--build-arg TENSORFLOW_VERSION=$(TENSORFLOW_VERSION) \
 		-t $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) && \
 	docker tag $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) $(DOCKER_PREFIX)-cuda-tf:$(GIT_MASTER_HEAD_SHA)
-	
+
 vnc: py
-	docker build docker/vnc \
+	docker buildx build --platform $(PLATFORMS) docker/vnc \
 		--build-arg BASE_IMAGE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		-t $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)$(RENKU_TAG) && \
 	docker tag $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-vnc:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)
 
 julia: py
-	docker build docker/julia \
+	docker buildx build --platform $(PLATFORMS) docker/julia \
 		--build-arg BASE_IMAGE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		-t $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)$(RENKU_TAG) && \
 	docker tag $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-julia:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)