From 6caf5c757e7b2d896bb056816a64ecab0b405e90 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 1 Aug 2024 13:08:49 +1000 Subject: [PATCH 1/3] MDBF-778 Allow building buildbot workers without a galera included To solve chicken/egg problem of providing galera workers to build the galera-4 package. nogalera defined as string due to https://github.com/actions/runner/issues/1483. It defaults to true, so the few build images that don't set it (mainly about to be EOL distros), or new distros that accidently omit it won't fail. --- .github/workflows/bbw_build_container_rhel.yml | 6 ++++++ .github/workflows/bbw_build_container_template.yml | 10 ++++++++-- .github/workflows/build-centos-based.yml | 4 ++++ .github/workflows/build-centos.pip-based.yml | 4 ++++ .github/workflows/build-debian-based.yml | 11 +++++++++++ .github/workflows/build-debian.msan-based.yml | 3 +++ .github/workflows/build-fedora-based.yml | 5 +++++ .github/workflows/build-opensuse.pip-based.yml | 3 +++ .github/workflows/build-sles.pip-based.yml | 3 +++ 9 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bbw_build_container_rhel.yml b/.github/workflows/bbw_build_container_rhel.yml index de09fd6c..218eb587 100644 --- a/.github/workflows/bbw_build_container_rhel.yml +++ b/.github/workflows/bbw_build_container_rhel.yml @@ -27,14 +27,17 @@ jobs: - dockerfile: rhel7.Dockerfile pip.Dockerfile image: rhel7 platforms: linux/amd64 + nogalera: false - dockerfile: rhel.Dockerfile image: ubi8 tag: rhel8 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x + nogalera: false - dockerfile: rhel.Dockerfile pip.Dockerfile image: ubi9 tag: rhel9 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x + nogalera: false env: BUILD_RHEL: false DEPLOY_IMAGES: false @@ -73,6 +76,9 @@ jobs: run: | cd ${{ env.WORKDIR }} cat ${{ matrix.dockerfile }} qpress.Dockerfile buildbot-worker.Dockerfile >$GITHUB_WORKSPACE/Dockerfile + if [ "${{ matrix.nogalera }}" == true ]; then + sed -i -e '/ci.mariadb.org\/galera/d' -e '/WSREP_PROVIDER/d' -e '/galera-4/d' $GITHUB_WORKSPACE/Dockerfile + fi cp -r qpress $GITHUB_WORKSPACE - name: Check Dockerfile with hadolint run: | diff --git a/.github/workflows/bbw_build_container_template.yml b/.github/workflows/bbw_build_container_template.yml index 80e29b74..c8a11ba2 100644 --- a/.github/workflows/bbw_build_container_template.yml +++ b/.github/workflows/bbw_build_container_template.yml @@ -31,6 +31,10 @@ on: files: required: false type: string + nogalera: + required: false + type: string + default: 'true' jobs: build: @@ -67,6 +71,9 @@ jobs: cd ${{ env.WORKDIR }} cat ${{ inputs.dockerfile }} qpress.Dockerfile buildbot-worker.Dockerfile >$GITHUB_WORKSPACE/Dockerfile cp -r qpress $GITHUB_WORKSPACE + if [ "${{ inputs.nogalera }}" == true ]; then + sed -i -e '/ci.mariadb.org\/galera/d' -e '/galera-4/d' -e '/WSREP_PROVIDER/d' $GITHUB_WORKSPACE/Dockerfile + fi - name: Copy files on runner if: ${{ inputs.files }} @@ -78,8 +85,7 @@ jobs: done - name: No wsrep on 32 bit platforms - if: > - (contains(inputs.platforms, 'linux/386')) + if: contains(inputs.platforms, 'linux/386') run: | sed -i -e '/WSREP_PROVIDER/d' $GITHUB_WORKSPACE/Dockerfile - name: Check Dockerfile with hadolint diff --git a/.github/workflows/build-centos-based.yml b/.github/workflows/build-centos-based.yml index ec91460a..a822db1c 100644 --- a/.github/workflows/build-centos-based.yml +++ b/.github/workflows/build-centos-based.yml @@ -25,11 +25,15 @@ jobs: include: - image: almalinux:8 platforms: linux/amd64, linux/arm64/v8 + nogalera: false - image: rockylinux:8 platforms: linux/amd64, linux/arm64/v8 + nogalera: false + uses: ./.github/workflows/bbw_build_container_template.yml with: dockerfile: centos.Dockerfile image: ${{ matrix.image }} platforms: ${{ matrix.platforms }} + nogalera: ${{ matrix.nogalera }} secrets: inherit diff --git a/.github/workflows/build-centos.pip-based.yml b/.github/workflows/build-centos.pip-based.yml index 0ece5a98..d0b64e82 100644 --- a/.github/workflows/build-centos.pip-based.yml +++ b/.github/workflows/build-centos.pip-based.yml @@ -28,14 +28,17 @@ jobs: include: - image: almalinux:9 platforms: linux/amd64, linux/arm64/v8 + nogalera: false - image: rockylinux:9 platforms: linux/amd64, linux/arm64/v8 + nogalera: false - image: quay.io/centos/centos:stream9 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le tag: centosstream9 runner: ubuntu-24.04 + nogalera: false uses: ./.github/workflows/bbw_build_container_template.yml with: @@ -44,4 +47,5 @@ jobs: platforms: ${{ matrix.platforms }} runner: ${{ matrix.runner }} tag: ${{ matrix.tag }} + nogalera: ${{ matrix.nogalera }} secrets: inherit diff --git a/.github/workflows/build-debian-based.yml b/.github/workflows/build-debian-based.yml index 74c5e0df..ff48caff 100644 --- a/.github/workflows/build-debian-based.yml +++ b/.github/workflows/build-debian-based.yml @@ -27,41 +27,51 @@ jobs: - image: debian:11 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le branch: 10.11 + nogalera: false - image: debian:12 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le branch: 10.11 tag: debian12 + nogalera: false - image: debian:12 platforms: linux/386 branch: 10.11 tag: debian12-386 + nogalera: false - image: debian:sid platforms: linux/amd64, linux/arm64/v8, linux/ppc64le branch: 10.11 + nogalera: false - image: debian:sid platforms: linux/386 branch: 10.11 tag: debiansid-386 + nogalera: false - image: ubuntu:20.04 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x branch: 10.11 + nogalera: false - image: ubuntu:22.04 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x branch: 10.11 + nogalera: false - image: ubuntu:23.10 platforms: linux/amd64, linux/arm64/v8 branch: 10.11 + nogalera: false - image: ubuntu:24.04 platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x branch: 10.11 + nogalera: false + uses: ./.github/workflows/bbw_build_container_template.yml with: @@ -70,4 +80,5 @@ jobs: platforms: ${{ matrix.platforms }} tag: ${{ matrix.tag }} branch: ${{ matrix.branch }} + nogalera: ${{ matrix.nogalera }} secrets: inherit diff --git a/.github/workflows/build-debian.msan-based.yml b/.github/workflows/build-debian.msan-based.yml index 0db38318..fc7298a8 100644 --- a/.github/workflows/build-debian.msan-based.yml +++ b/.github/workflows/build-debian.msan-based.yml @@ -30,12 +30,14 @@ jobs: branch: 10.11 tag: debian11-msan clang_version: 15 + nogalera: false - image: debian:11 platforms: linux/amd64 branch: 10.11 tag: debian11-msan-clang-16 clang_version: 16 + nogalera: false uses: ./.github/workflows/bbw_build_container_template.yml with: @@ -45,4 +47,5 @@ jobs: tag: ${{ matrix.tag }} branch: ${{ matrix.branch }} clang_version: ${{ matrix.clang_version }} + nogalera: ${{ matrix.nogalera }} secrets: inherit diff --git a/.github/workflows/build-fedora-based.yml b/.github/workflows/build-fedora-based.yml index e6fa6c7a..4a5a573c 100644 --- a/.github/workflows/build-fedora-based.yml +++ b/.github/workflows/build-fedora-based.yml @@ -25,17 +25,21 @@ jobs: include: - image: fedora:38 platforms: linux/amd64, linux/arm64/v8 + nogalera: false - image: fedora:39 platforms: linux/amd64, linux/arm64/v8 + nogalera: false - image: fedora:40 platforms: linux/amd64, linux/arm64/v8 + nogalera: false - image: fedora:40 platforms: linux/amd64 tag: fedora40-valgrind install_valgrind: "true" + nogalera: false uses: ./.github/workflows/bbw_build_container_template.yml with: @@ -44,4 +48,5 @@ jobs: platforms: ${{ matrix.platforms }} tag: ${{ matrix.tag }} install_valgrind: ${{ matrix.install_valgrind }} + nogalera: ${{ matrix.nogalera }} secrets: inherit diff --git a/.github/workflows/build-opensuse.pip-based.yml b/.github/workflows/build-opensuse.pip-based.yml index d99a4807..587650bd 100644 --- a/.github/workflows/build-opensuse.pip-based.yml +++ b/.github/workflows/build-opensuse.pip-based.yml @@ -30,12 +30,15 @@ jobs: - image: opensuse/leap:15.6 platforms: linux/amd64 tag: opensuse15 + nogalera: false + uses: ./.github/workflows/bbw_build_container_template.yml with: dockerfile: opensuse.Dockerfile pip.Dockerfile image: ${{ matrix.image }} platforms: ${{ matrix.platforms }} tag: ${{ matrix.tag }} + nogalera: ${{ matrix.nogalera }} files: '[ {"name": "mariadb_zypper_expect", "source": "''$WORKDIR/mariadb_zypper_expect''", "target": "''$GITHUB_WORKSPACE''"} diff --git a/.github/workflows/build-sles.pip-based.yml b/.github/workflows/build-sles.pip-based.yml index 05318972..9e1508e8 100644 --- a/.github/workflows/build-sles.pip-based.yml +++ b/.github/workflows/build-sles.pip-based.yml @@ -31,12 +31,15 @@ jobs: - image: registry.suse.com/bci/bci-base:15.6 platforms: linux/amd64, linux/s390x tag: sles15 + nogalera: false + uses: ./.github/workflows/bbw_build_container_template.yml with: dockerfile: sles.Dockerfile pip.Dockerfile image: ${{ matrix.image }} platforms: ${{ matrix.platforms }} tag: ${{ matrix.tag }} + nogalera: ${{ matrix.nogalera }} files: '[ {"name": "mariadb_zypper_expect", "source": "''$WORKDIR/mariadb_zypper_expect''", "target": "''$GITHUB_WORKSPACE''"} From e47dd8399e8754f5954d40ff1ac5db21c071ca47 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 22 Aug 2024 15:07:07 +1000 Subject: [PATCH 2/3] MDBF-778: Allow building buildbot workers without a galera included To support the nongalera workflow the galera removed lines must be workable when those lines are removed. This means if statements cannot be empty and there should be no lingering `fi` closures. --- ci_build_images/debian.Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci_build_images/debian.Dockerfile b/ci_build_images/debian.Dockerfile index 5d911184..4dd8e30f 100644 --- a/ci_build_images/debian.Dockerfile +++ b/ci_build_images/debian.Dockerfile @@ -37,8 +37,7 @@ RUN . /etc/os-release \ && if [ "${VERSION_ID}" = "20.04" ]; then apt-get -y install --no-install-recommends g++-10; fi \ && if [ "$(arch)" = "x86_64" ]; then ARCH="amd64"; else ARCH=$(arch); fi \ && if curl --head --silent "https://ci.mariadb.org/galera/mariadb-4.x-latest-gal-${ARCH}-${ID}-$(echo "$VERSION_ID" | sed 's/\.//').sources" | head -n1 | grep -q 200; then \ - curl -s "https://ci.mariadb.org/galera/mariadb-4.x-latest-gal-${ARCH}-${ID}-$(echo "$VERSION_ID" | sed 's/\.//').sources" >/etc/apt/sources.list.d/galera-4.sources; \ - fi \ + curl -s "https://ci.mariadb.org/galera/mariadb-4.x-latest-gal-${ARCH}-${ID}-$(echo "$VERSION_ID" | sed 's/\.//').sources" >/etc/apt/sources.list.d/galera-4.sources; fi \ && apt-get update \ && curl -skO https://raw.githubusercontent.com/MariaDB/server/44e4b93316be8df130c6d87880da3500d83dbe10/debian/control \ && mkdir debian \ @@ -82,9 +81,7 @@ RUN . /etc/os-release \ socat \ sudo \ wget \ - && if [ "$(getconf LONG_BIT)" = 64 ]; then \ - apt-get -y install --no-install-recommends galera-4; \ - fi \ + && if [ "$(getconf LONG_BIT)" = 64 ]; then apt-get -y install --no-install-recommends galera-4; fi \ && if [ "${VERSION_ID}" != 18.04 ]; then \ apt-get -y install --no-install-recommends flex; \ fi \ From f1b58abf07d781d8b242b51f135bf44546a2040e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 22 Aug 2024 16:20:06 +1000 Subject: [PATCH 3/3] MDBF-778: no-galera - adjust debian.Dockerfile to remove ARCH= This ensure that hadolint doesn't fail with: /mnt/Dockerfile:28 SC2034 warning: ARCH appears unused. Verify use (or export if used externally). --- ci_build_images/debian.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_build_images/debian.Dockerfile b/ci_build_images/debian.Dockerfile index 4dd8e30f..e47f903a 100644 --- a/ci_build_images/debian.Dockerfile +++ b/ci_build_images/debian.Dockerfile @@ -35,7 +35,7 @@ RUN . /etc/os-release \ equivs \ lsb-release \ && if [ "${VERSION_ID}" = "20.04" ]; then apt-get -y install --no-install-recommends g++-10; fi \ - && if [ "$(arch)" = "x86_64" ]; then ARCH="amd64"; else ARCH=$(arch); fi \ + && if [ "$(arch)" = "x86_64" ]; then ARCH="amd64"; else ARCH=$(arch); echo /* galera-4 */; fi \ && if curl --head --silent "https://ci.mariadb.org/galera/mariadb-4.x-latest-gal-${ARCH}-${ID}-$(echo "$VERSION_ID" | sed 's/\.//').sources" | head -n1 | grep -q 200; then \ curl -s "https://ci.mariadb.org/galera/mariadb-4.x-latest-gal-${ARCH}-${ID}-$(echo "$VERSION_ID" | sed 's/\.//').sources" >/etc/apt/sources.list.d/galera-4.sources; fi \ && apt-get update \