Rework release workflow for release/2.60 branch (#12449)

Cherry pick #12331 into `release/2.61` Co-authored-by: lystopad <[email protected]>
erigontech · Oct 24, 2024 · 8de7eae · 8de7eae
1 parent 1b49666
commit 8de7eae
Show file tree

Hide file tree

Showing 2 changed files with 131 additions and 106 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -2,26 +2,15 @@ name: Release
 
 env:
   APPLICATION: "erigon"
-  BUILDER_IMAGE: "ghcr.io/goreleaser/goreleaser-cross:v1.22.7"
-  DOCKER_BASE_IMAGE: "alpine:3.20.3"
+  BUILDER_IMAGE: "golang:1.22-bookworm"
+  DOCKER_BASE_IMAGE: "debian:12.7-slim"
   APP_REPO: "erigontech/erigon"
   PACKAGE: "github.com/erigontech/erigon"
   DOCKERHUB_REPOSITORY: "erigontech/erigon"
-  DOCKERFILE_PATH: "./Dockerfile.release"
-  GITHUB_AUTOMATION_EMAIL: "[email protected]"
-  GITHUB_AUTOMATION_NAME: "Erigon Github Automation"
+  DOCKERFILE_PATH: "Dockerfile.release"
   LABEL_DESCRIPTION: "Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency frontier. Archive Node by default."
 
 on:
-  push:
-    branches-ignore:
-      - '**'
-    #branches:
-    #  - 'master'
-    #tags:
-      ## only trigger on release tags:
-      #- 'v*.*.*'
-      #- 'v*.*.*-*'
   workflow_dispatch:
     inputs:
       checkout_ref:
@@ -32,7 +21,7 @@ on:
       release_version:
         required: true
         type: string
-        description: 'Release version number (Pattern - #.#.# , f.e. 2.41.3 or 3.0.0 or 3.0.0-alpha1 for pre-releases. Do not prefix it with "v".)'
+        description: 'Release version number (Pattern - v#.#.# , f.e. v2.60.1 or v3.0.0 or v3.0.0-alpha1 for pre-releases. Use prefix "v".)'
       perform_release:
         required: false
         type: boolean
@@ -48,8 +37,8 @@ on:
 jobs:
 
   build-release:
-    ## runs-on: ubuntu-22.04
     runs-on: ubuntu-latest-devops-xxlarge
+    # runs-on: ubuntu-latest
     timeout-minutes: 60
     name: Build Artifacts and multi-platform Docker image, publish draft of the Release Notes
 
@@ -60,6 +49,7 @@ jobs:
           repository: ${{ env.APP_REPO }}
           fetch-depth: 0
           ref: ${{ inputs.checkout_ref }}
+          path: 'erigon'
 
       - name: Check if tag ${{ inputs.release_version }} already exists in case perform_release is set.
         if: ${{ (inputs.perform_release) && (inputs.release_version != '') }}
@@ -71,11 +61,13 @@ jobs:
             echo "OK: tag ${{ inputs.release_version }} does not exists. Proceeding."
           fi
 
-      - name: Get commit id
+      - name: Run some commands, get commit id
         id: getCommitId
         run: |
+          mkdir $GITHUB_WORKSPACE/build-arm64 $GITHUB_WORKSPACE/build-amd64 $GITHUB_WORKSPACE/build-amd64v2 
           echo "id=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
           echo "short_commit_id=$(git rev-parse --short=7 HEAD)" >> $GITHUB_OUTPUT
+          echo "week_of_the_year=$(/bin/date -u "+%Y-%W")" >> $GITHUB_OUTPUT
 
       - name: Login to Docker Hub
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567  ## v3.3.0
@@ -89,26 +81,77 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db ## v3.6.1
 
-      - name: Build binaries with goreleaser
+      - name: Setup GO build and pkg cache for one week only
+        id: cache
+        uses: actions/cache@v4
+        with:
+          key: cache-year-week-${{ steps.getCommitId.outputs.week_of_the_year }}-go.mod-hash:${{ hashFiles('erigon/go.mod') }}
+          path: |
+            ~/go/pkg
+            ~/.cache
+
+      - name: Build for linux/arm64
+        run: >
+          docker run --platform linux/arm64
+          --rm
+          -v $(pwd)/erigon:/erigon:ro
+          -v ${GITHUB_WORKSPACE}/build-arm64:/erigon-build
+          -v ${HOME}/.cache:/root/.cache
+          -v ${HOME}/go/pkg/mod:/go/pkg/mod
+          -w /erigon --entrypoint /bin/bash
+          ${{ env.BUILDER_IMAGE }}
+          -c "git config --global --add safe.directory /erigon;
+          make GOARCH=arm64 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb,nosilkworm 
+          erigon downloader devnet evm caplin diag integration rpcdaemon sentry txpool"
+
+      - name: Build for linux/amd64
+        run: >
+          docker run --platform linux/amd64
+          --rm
+          -v $(pwd)/erigon:/erigon:ro
+          -v ${GITHUB_WORKSPACE}/build-amd64:/erigon-build
+          -v ${HOME}/.cache:/root/.cache
+          -v ${HOME}/go/pkg/mod:/go/pkg/mod
+          -w /erigon --entrypoint /bin/bash
+          ${{ env.BUILDER_IMAGE }}
+          -c "git config --global --add safe.directory /erigon;
+          make GOARCH=amd64 GOAMD64=v1 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb 
+          erigon downloader devnet evm caplin diag integration rpcdaemon sentry txpool;
+          find / -name libsilkworm_capi.so -exec install {} /erigon-build \; "
+
+      - name: Build for linux/amd64/v2
+        run: >
+          docker run --platform linux/amd64/v2
+          --rm
+          -v $(pwd)/erigon:/erigon:ro
+          -v ${GITHUB_WORKSPACE}/build-amd64v2:/erigon-build
+          -v ${HOME}/.cache:/root/.cache
+          -v ${HOME}/go/pkg/mod:/go/pkg/mod
+          -w /erigon --entrypoint /bin/bash
+          ${{ env.BUILDER_IMAGE }}
+          -c "git config --global --add safe.directory /erigon;
+          make GOARCH=amd64 GOAMD64=v2 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb 
+          erigon downloader devnet evm caplin diag integration rpcdaemon sentry txpool;
+          find / -name libsilkworm_capi.so -exec install {} /erigon-build \; "
+
+      - name: Create archives and checksums
         env:
-          BUILD_VERSION: ${{ inputs.release_version }}
-          DOCKER_URL: ${{ env.DOCKERHUB_REPOSITORY }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
         run: |
-          docker run --rm \
-            -w /${{ env.APPLICATION }}/ \
-            -e BUILD_VERSION=${{ env.BUILD_VERSION }} \
-            -e GIT_COMMIT=${{ steps.getCommitId.outputs.id }} \
-            -e GIT_BRANCH=${{ inputs.checkout_ref }} \
-            -e GIT_TAG=${{ inputs.release_version }} \
-            -e PACKAGE=${{ env.PACKAGE }} \
-            -e APPLICATION=${{ env.APPLICATION }} \
-            -v $(pwd):/${{ env.APPLICATION}} \
-            -v /var/run/docker.sock:/var/run/docker.sock \
-            ${{ env.BUILDER_IMAGE }} release --timeout 60m0s --clean --skip=validate,announce,publish
-          echo "DEBUG: ls -lao in the working directory"
-          ls -lao
-          echo "DEBUG: content of the dist/ directory"
-          find dist/ -ls
+          cd ${GITHUB_WORKSPACE}
+          mkdir $GITHUB_WORKSPACE/release
+          for dir in build-*; do
+            cd $dir
+            echo Current directory is $(pwd) . Checksum file and archive will be created for this directory
+            sha256sum * > checksums.txt
+            tar czvf $GITHUB_WORKSPACE/release/${APPLICATION}_${RELEASE_VERSION}_linux_$(echo $dir | sed 's,build-,,').tar.gz \
+              --transform "s,^./,${APPLICATION}_${RELEASE_VERSION}_linux_$(echo $dir | sed 's,build-,,')/," .
+            cd -
+          done
+          cd $GITHUB_WORKSPACE/release
+          sha256sum * > ${APPLICATION}_${RELEASE_VERSION}_checksums.txt
+          echo Content of release directory:
+          find . -type f -ls
 
       - name: Build and push multi-platform docker images (${{ env.BUILD_VERSION }} and latest) in case perform_release is true
         if: ${{ inputs.perform_release }}
@@ -117,35 +160,37 @@ jobs:
           DOCKER_URL: ${{ env.DOCKERHUB_REPOSITORY }}
           DOCKER_PUBLISH_LATEST_CONDITION: ${{ inputs.publish_latest_tag && format('--tag {0}:latest ',env.DOCKERHUB_REPOSITORY) || '' }}
         run: |
-          docker buildx build \
-          --file ${{ env.DOCKERFILE_PATH }} \
-          --build-arg RELEASE_DOCKER_BASE_IMAGE=${{ env.DOCKER_BASE_IMAGE }} \
-          --build-arg VERSION=${{ env.BUILD_VERSION }} \
-          --build-arg APPLICATION=${{ env.APPLICATION }} \
-          --tag ${{ env.DOCKER_URL }}:${{ env.BUILD_VERSION }} \
-          --target release \
-          --attest type=provenance,mode=max \
-          --sbom=true \
-          ${{ env.DOCKER_PUBLISH_LATEST_CONDITION }} \
-          --label org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
-          --label org.opencontainers.image.authors="https://github.com/erigontech/erigon/graphs/contributors" \
-          --label org.opencontainers.image.url="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
-          --label org.opencontainers.image.documentation="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
-          --label org.opencontainers.image.source="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
-          --label org.opencontainers.image.version=${{ inputs.release_version }} \
-          --label org.opencontainers.image.revision=${{ steps.getCommitId.outputs.id }} \
-          --label org.opencontainers.image.vcs-ref-short=${{ steps.getCommitId.outputs.short_commit_id }} \
-          --label org.opencontainers.image.vendor="${{ github.repository_owner }}" \
-          --label org.opencontainers.image.description="${{ env.LABEL_DESCRIPTION }}" \
-          --label org.opencontainers.image.base.name="${{ env.DOCKER_BASE_IMAGE }}" \
-          --push \
-          --platform linux/amd64,linux/amd64/v2,linux/arm64 .
+            cd ${GITHUB_WORKSPACE}/release
+            echo "Current directory is $(pwd) ."
+            docker buildx build \
+            --file ${{ github.workspace }}/erigon/${{ env.DOCKERFILE_PATH }} \
+            --build-arg RELEASE_DOCKER_BASE_IMAGE=${{ env.DOCKER_BASE_IMAGE }} \
+            --build-arg VERSION=${{ env.BUILD_VERSION }} \
+            --build-arg APPLICATION=${{ env.APPLICATION }} \
+            --tag ${{ env.DOCKER_URL }}:${{ env.BUILD_VERSION }} \
+            ${{ env.DOCKER_PUBLISH_LATEST_CONDITION }} \
+            --target release \
+            --attest type=provenance,mode=max \
+            --sbom=true \
+            --label org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+            --label org.opencontainers.image.authors="https://github.com/erigontech/erigon/graphs/contributors" \
+            --label org.opencontainers.image.url="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
+            --label org.opencontainers.image.documentation="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
+            --label org.opencontainers.image.source="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
+            --label org.opencontainers.image.version=${{ inputs.release_version }} \
+            --label org.opencontainers.image.revision=${{ steps.getCommitId.outputs.id }} \
+            --label org.opencontainers.image.vcs-ref-short=${{ steps.getCommitId.outputs.short_commit_id }} \
+            --label org.opencontainers.image.vendor="${{ github.repository_owner }}" \
+            --label org.opencontainers.image.description="${{ env.LABEL_DESCRIPTION }}" \
+            --label org.opencontainers.image.base.name="${{ env.DOCKER_BASE_IMAGE }}" \
+            --push \
+            --platform linux/amd64,linux/amd64/v2,linux/arm64 .
 
       - name: Upload artifact -- linux/arm64
         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  ## v4.3.6
         with:
           name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_arm64.tar.gz
-          path: ./dist/${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_arm64.tar.gz
+          path: ${{ github.workspace }}/release/${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_arm64.tar.gz
           retention-days: 1
           compression-level: 0
           if-no-files-found: error
@@ -154,7 +199,7 @@ jobs:
         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  ## v4.3.6
         with:
           name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64.tar.gz
-          path: ./dist/${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64.tar.gz
+          path: ${{ github.workspace }}/release/${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64.tar.gz
           retention-days: 1
           compression-level: 0
           if-no-files-found: error
@@ -163,60 +208,25 @@ jobs:
         uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  ## v4.3.6
         with:
           name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64v2.tar.gz
-          path: ./dist/${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64v2.tar.gz
-          retention-days: 1
-          compression-level: 0
-          if-no-files-found: error
-
-      - name: Upload artifact -- darwin/arm64
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  ## v4.3.6
-        with:
-          name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_darwin_arm64.tar.gz
-          path: ./dist/${{ env.APPLICATION }}_${{ inputs.release_version }}_darwin_arm64.tar.gz
-          retention-days: 1
-          compression-level: 0
-          if-no-files-found: error
-
-      - name: Upload artifact -- darwin/amd64
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  ## v4.3.6
-        with:
-          name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_darwin_amd64.tar.gz
-          path: ./dist/${{ env.APPLICATION }}_${{ inputs.release_version }}_darwin_amd64.tar.gz
+          path: ${{ github.workspace }}/release/${{ env.APPLICATION }}_${{ inputs.release_version }}_linux_amd64v2.tar.gz
           retention-days: 1
           compression-level: 0
           if-no-files-found: error
 
-      - name: Upload artifact -- windows/amd64
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a  ## v4.3.6
-        with:
-          name: ${{ env.APPLICATION }}_${{ inputs.release_version }}_windows_amd64.zip
-          path: ./dist/${{ env.APPLICATION }}_${{ inputs.release_version }}_windows_amd64.zip
-          retention-days: 1
-          compression-level: 0
-          if-no-files-found: error
-
-## not required for now -- commented:
-#      - name: Create and push a git tag for the released version in case perform_release is set
-#        if: ${{ inputs.perform_release }}
-#        run: |
-#          git config --global user.email ${{ env.GITHUB_AUTOMATION_EMAIL }}
-#          git config --global user.name "${{ env.GITHUB_AUTOMATION_NAME }}"
-#          git tag -a ${{ inputs.release_version }} -m "Release ${{ inputs.release_version }}"
-#          git push origin ${{ inputs.release_version }}
-
-      - name: Publish draft of the Release notes with assets (without windows .zip) in case perform_release is set
+      - name: Publish draft of the Release Notes with assets in case perform_release is set
         if: ${{ inputs.perform_release }}
         env:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           DOCKER_TAGS: ${{ env.DOCKERHUB_REPOSITORY }}:${{ inputs.release_version }}
           GITHUB_RELEASE_TARGET: ${{ inputs.checkout_ref }}
         run: |
-          cd dist
-          sha256sum *.tar.gz > ${HOME}/${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt
-          gh release create ${{ inputs.release_version }} *.tar.gz ${HOME}/${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt \
+          cd $GITHUB_WORKSPACE/release
+          gh release create \
             --generate-notes \
             --target ${GITHUB_RELEASE_TARGET} \
             --draft=true \
             --title "${{ inputs.release_version }}" \
-            --notes "**Improvements:**<br>- ...coming soon <br><br>**Bugfixes:**<br><br>- ...coming soon<br><br>**Docker images:**<br><br>Docker image released:<br> ${{ env.DOCKER_TAGS }}<br><br>... coming soon<br>"
+            --notes "**Improvements:**<br>- ...coming soon <br><br>**Bugfixes:**<br><br>- ...coming soon<br><br>**Docker images:**<br><br>Docker image released:<br> ${{ env.DOCKER_TAGS }}<br><br>... coming soon<br>" \
+             ${{ inputs.release_version }} \
+             *.tar.gz ${{ env.APPLICATION }}_${{ inputs.release_version }}_checksums.txt
diff --git a/Dockerfile.release b/Dockerfile.release
@@ -22,7 +22,8 @@ ARG TARGETARCH \
     VERSION=${VERSION} \
     APPLICATION
 
-COPY ./dist/${APPLICATION}_${VERSION}_linux_${TARGETARCH}${TARGETVARIANT}.tar.gz /tmp/${APPLICATION}.tar.gz
+COPY ${APPLICATION}_${VERSION}_linux_${TARGETARCH}${TARGETVARIANT}.tar.gz /tmp/${APPLICATION}.tar.gz
+
 RUN tar xzvf /tmp/${APPLICATION}.tar.gz -C /tmp && \
     mv /tmp/${APPLICATION}_${VERSION}_linux_${TARGETARCH}${TARGETVARIANT} /tmp/${APPLICATION}
 
@@ -31,12 +32,26 @@ FROM ${RELEASE_DOCKER_BASE_IMAGE} AS release
 ARG USER=erigon \
     GROUP=erigon \
     APPLICATION \
+    TARGETARCH \
     EXPOSED_PORTS
 
+STOPSIGNAL 2
+
+SHELL ["/bin/bash", "-c"]
+
 RUN --mount=type=bind,from=temporary,source=/tmp/${APPLICATION},target=/tmp/${APPLICATION} \
-    apk add --no-cache ca-certificates tzdata && \
-    addgroup ${GROUP} && \
-    adduser -D -h /home/${USER} -G ${GROUP} ${USER} && \
+    echo Installing on ${TARGETOS} with variant ${TARGETVARIANT} && \
+    adduser --group ${GROUP} && \
+    adduser --system --ingroup ${GROUP} --home /home/${USER} --shell /bin/bash ${USER} && \
+    apt update -y && \
+    apt install -y --no-install-recommends ca-certificates && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    if [ "x${TARGETARCH}" == "xamd64" ]; then \
+        echo "Installing libsilkworm_capi.so library to /lib/x86_64-linux-gnu/ in case amd64 architecture:"; \
+        find /tmp/${APPLICATION} -name libsilkworm_capi.so -type f | xargs -I % install -m a=r -v % /lib/x86_64-linux-gnu/; \
+        echo "Done." ; \
+    fi && \    
     install -d -o ${USER} -g ${GROUP} /home/${USER}/.local /home/${USER}/.local/share /home/${USER}/.local/share/erigon && \
     install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/erigon /usr/local/bin/ && \
     install -o ${USER} -g ${GROUP} /tmp/${APPLICATION}/integration /usr/local/bin/ && \