From e2523978fa4a92310d0f62561a86976564b88f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 26 Feb 2024 17:21:09 +0100 Subject: [PATCH 01/11] add Dockerfile for debian 12 build container --- .../Dockerfile.EESSI-build-node-debian12 | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 containers/Dockerfile.EESSI-build-node-debian12 diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 new file mode 100644 index 00000000..b4d70ce1 --- /dev/null +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -0,0 +1,45 @@ +ARG cvmfsversion=2.11.2 +ARG awscliversion=1.32.22 +ARG fuseoverlayfsversion=1.10 + +FROM debian:12.5 AS prepare-deb +ARG cvmfsversion +COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh +RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion} + + +FROM debian:12.5 +ARG cvmfsversion +ARG awscliversion +ARG fuseoverlayfsversion + +COPY --from=prepare-deb /root/deb /root/deb + +RUN apt-get update +RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace +# python3 and jq are required for eessi-upload-to-staging script (next to awscli) +RUN apt-get install -y python3-pip jq +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-config-default_latest_all.deb \ + /root/deb/cvmfs-config-eessi_latest_all.deb + +# download binary for specific version of fuse-overlayfs +RUN curl -L -o /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/v${fuseoverlayfsversion}/fuse-overlayfs-$(uname -m) \ + && chmod +x /usr/local/bin/fuse-overlayfs + +RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \ + && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \ + && echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local + +RUN mkdir -p /cvmfs/software.eessi.io + +RUN useradd -ms /bin/bash eessi + +# stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) +RUN pip3 install archspec awscli==${awscliversion} + +RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ + && mv eessi-upload-to-staging /usr/bin \ + && chmod a+x /usr/bin/eessi-upload-to-staging From 5679f2d75e4ac8c03b7f5189ea61681a22312a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 26 Feb 2024 17:24:03 +0100 Subject: [PATCH 02/11] add build-node:debian12 --- .github/workflows/build-publish-containers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index ec2bcd9f..527696dd 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -32,7 +32,7 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11'] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian12'] platform: [amd64, arm64, ppc64le] steps: @@ -91,7 +91,7 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11'] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian12'] steps: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' From 79c310b6927d56d008e21cdf5778dbad9fbee2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 09:33:02 +0100 Subject: [PATCH 03/11] drop ppc64le support --- .github/workflows/build-publish-containers.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index 527696dd..411ee5cd 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -33,7 +33,7 @@ jobs: strategy: matrix: tag: ['client:centos7', 'build-node:debian11', 'build-node:debian12'] - platform: [amd64, arm64, ppc64le] + platform: [amd64, arm64] steps: - name: Check out the repo @@ -110,8 +110,7 @@ jobs: run: | docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \ ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \ - ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \ - ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le + ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 - name: Inspect the final image if: github.event_name != 'pull_request' From c039d66fe21f3bbfd0bb4106658166113c1620fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 13:14:48 +0100 Subject: [PATCH 04/11] install archspec and awscli in virtual environment --- containers/Dockerfile.EESSI-build-node-debian12 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index b4d70ce1..3bcc4e45 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -18,7 +18,7 @@ COPY --from=prepare-deb /root/deb /root/deb RUN apt-get update RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace # python3 and jq are required for eessi-upload-to-staging script (next to awscli) -RUN apt-get install -y python3-pip jq +RUN apt-get install -y python3-pip python3-venv jq RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ @@ -37,8 +37,9 @@ RUN mkdir -p /cvmfs/software.eessi.io RUN useradd -ms /bin/bash eessi +RUN python3 -m venv /opt/archspec && source /opt/archspec/bin/activate && pip3 install archspec && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) -RUN pip3 install archspec awscli==${awscliversion} +RUN python3 -m venv /opt/awscli && source /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ && mv eessi-upload-to-staging /usr/bin \ From 256d7ef84b6c92b1564a83336f88761fa4068133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 13:24:39 +0100 Subject: [PATCH 05/11] use '.' instead of 'source' --- containers/Dockerfile.EESSI-build-node-debian12 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index 3bcc4e45..e591184e 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -37,9 +37,9 @@ RUN mkdir -p /cvmfs/software.eessi.io RUN useradd -ms /bin/bash eessi -RUN python3 -m venv /opt/archspec && source /opt/archspec/bin/activate && pip3 install archspec && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec +RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) -RUN python3 -m venv /opt/awscli && source /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws +RUN python3 -m venv /opt/awscli && . /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ && mv eessi-upload-to-staging /usr/bin \ From 141afc2c958935d27397a95bb7399282e85e9481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 13:26:46 +0100 Subject: [PATCH 06/11] pin archspec version --- containers/Dockerfile.EESSI-build-node-debian11 | 4 +++- containers/Dockerfile.EESSI-build-node-debian12 | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian11 b/containers/Dockerfile.EESSI-build-node-debian11 index 4fe34ae3..8e1a3b8a 100644 --- a/containers/Dockerfile.EESSI-build-node-debian11 +++ b/containers/Dockerfile.EESSI-build-node-debian11 @@ -1,4 +1,5 @@ ARG cvmfsversion=2.11.2 +ARG archspecversion=0.2.2 ARG awscliversion=1.32.22 ARG fuseoverlayfsversion=1.10 @@ -10,6 +11,7 @@ RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion} FROM debian:11.5 ARG cvmfsversion +ARG archspecversion ARG awscliversion ARG fuseoverlayfsversion @@ -38,7 +40,7 @@ RUN mkdir -p /cvmfs/software.eessi.io RUN useradd -ms /bin/bash eessi # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) -RUN pip3 install archspec awscli==${awscliversion} +RUN pip3 install archspec==${archspecversion} awscli==${awscliversion} RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ && mv eessi-upload-to-staging /usr/bin \ diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index e591184e..7c1c7f0a 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -1,4 +1,5 @@ ARG cvmfsversion=2.11.2 +ARG archspecversion=0.2.2 ARG awscliversion=1.32.22 ARG fuseoverlayfsversion=1.10 @@ -10,6 +11,7 @@ RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion} FROM debian:12.5 ARG cvmfsversion +ARG archspecversion ARG awscliversion ARG fuseoverlayfsversion @@ -37,7 +39,7 @@ RUN mkdir -p /cvmfs/software.eessi.io RUN useradd -ms /bin/bash eessi -RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec +RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec==${archspecversion} && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) RUN python3 -m venv /opt/awscli && . /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws From 9a4bd68185c896a689bf6d2e9a36ca17898000af Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 27 Feb 2024 14:01:34 +0100 Subject: [PATCH 07/11] Make sure Debian 12 is used throughout --- containers/Dockerfile.EESSI-build-node-debian12 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index 7c1c7f0a..ede36526 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -21,9 +21,9 @@ RUN apt-get update RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace # python3 and jq are required for eessi-upload-to-staging script (next to awscli) RUN apt-get install -y python3-pip python3-venv jq -RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-config-default_latest_all.deb \ /root/deb/cvmfs-config-eessi_latest_all.deb From 68604efe15670b7830f34da46be4057e637f4a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 14:01:48 +0100 Subject: [PATCH 08/11] fix cvmfs package names for debian 12 --- containers/Dockerfile.EESSI-build-node-debian12 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index 7c1c7f0a..ede36526 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -21,9 +21,9 @@ RUN apt-get update RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace # python3 and jq are required for eessi-upload-to-staging script (next to awscli) RUN apt-get install -y python3-pip python3-venv jq -RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-config-default_latest_all.deb \ /root/deb/cvmfs-config-eessi_latest_all.deb From 5ae63786ea65c61efa959f0fdde51712e63cf433 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 27 Feb 2024 14:07:59 +0100 Subject: [PATCH 09/11] Revert hasty change --- containers/Dockerfile.EESSI-build-node-debian12 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index ede36526..7c1c7f0a 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -21,9 +21,9 @@ RUN apt-get update RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace # python3 and jq are required for eessi-upload-to-staging script (next to awscli) RUN apt-get install -y python3-pip python3-venv jq -RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-libs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-config-default_latest_all.deb \ /root/deb/cvmfs-config-eessi_latest_all.deb From 0e3d3de9b07888f67141c651dd54bf7773c4ca0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 14:17:27 +0100 Subject: [PATCH 10/11] dynamically determine os --- containers/build-or-download-cvmfs-debs.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 071fa867..decde578 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -1,10 +1,12 @@ #/bin/bash cvmfsversion=$1 -arch=$(dpkg --print-architecture) -os=debian11 apt-get update -apt-get install -y wget +apt-get install -y wget lsb-release + +arch=$(dpkg --print-architecture) +os="$(lsb_release -si | tr [:upper:] [:lower:])$(lsb_release -sr)" + if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ] then apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools python3-dev python3-setuptools uuid-dev valgrind libz-dev lsb-release From 310b69d93fbaf4779f74187796c146198a22dec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 27 Feb 2024 14:17:49 +0100 Subject: [PATCH 11/11] use debian12 in package names --- containers/Dockerfile.EESSI-build-node-debian12 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian12 b/containers/Dockerfile.EESSI-build-node-debian12 index 7c1c7f0a..ede36526 100644 --- a/containers/Dockerfile.EESSI-build-node-debian12 +++ b/containers/Dockerfile.EESSI-build-node-debian12 @@ -21,9 +21,9 @@ RUN apt-get update RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace # python3 and jq are required for eessi-upload-to-staging script (next to awscli) RUN apt-get install -y python3-pip python3-venv jq -RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ - /root/deb/cvmfs-libs_${cvmfsversion}~1+debian11_$(dpkg --print-architecture).deb \ +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \ /root/deb/cvmfs-config-default_latest_all.deb \ /root/deb/cvmfs-config-eessi_latest_all.deb