From 38f0edf49dc46449e3381af9c7e958d580e44855 Mon Sep 17 00:00:00 2001 From: Til Blechschmidt Date: Mon, 22 Mar 2021 20:26:41 +0100 Subject: [PATCH] :construction_worker: Coalesce integration test and release pipeline --- .../workflows/{integration.yml => build.yml} | 156 +++++++++- .github/workflows/release.yml | 293 ------------------ 2 files changed, 153 insertions(+), 296 deletions(-) rename .github/workflows/{integration.yml => build.yml} (60%) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/build.yml similarity index 60% rename from .github/workflows/integration.yml rename to .github/workflows/build.yml index 06ab5e75..47896b2a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/build.yml @@ -1,11 +1,15 @@ -name: 🔎 Integration tests +name: 🚀 Project build -on: [push] +on: + push: + release: + types: [published] env: CARGO_TERM_COLOR: always jobs: + # ------------------ COMMON JOBS ------------------ build-core: name: 🔨 Build Core runs-on: ubuntu-latest @@ -15,9 +19,15 @@ jobs: with: path: | core/.cache - key: build-core-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + key: build-core-${{ github.event_name }}-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + - name: Build core in debug configuration + if: github.event_name == 'push' run: make core-debug + - name: Build core in release configuration + if: github.event_name == 'release' + run: make core + - name: Fix permission for cache run: sudo chmod -R 777 core/.cache - uses: actions/upload-artifact@v2 @@ -227,3 +237,143 @@ jobs: kubectl get services python3 test/integration.py 30007 + + # ------------------ RELEASE ONLY JOBS ------------------ + docker-hub: + if: github.event_name == 'release' + name: 🐳 Publish Images to DockerHub + runs-on: ubuntu-latest + needs: + - bundle-api + - bundle-core + - bundle-node + - kubernetes-integration + - docker-integration + steps: + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: webgrid + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - name: Prepare environment + run: | + echo "SRC_REPOSITORY=ghcr.io/tilblechschmidt/webgrid" >> $GITHUB_ENV + echo "SRC_TAG=sha-${GITHUB_SHA::8}" >> $GITHUB_ENV + + echo "DST_REPOSITORY=webgrid" >> $GITHUB_ENV + echo "DST_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + - name: Cross-push core + run: | + SRC_IMAGE=${{ env.SRC_REPOSITORY }}/core:${{ env.SRC_TAG }} + DST_IMAGE=${{ env.DST_REPOSITORY }}/core:${{ env.DST_TAG }} + + echo "Pushing $SRC_IMAGE to $DST_IMAGE" + docker pull $SRC_IMAGE + docker tag $SRC_IMAGE $DST_IMAGE + docker push $DST_IMAGE + - name: Cross-push api + run: | + SRC_IMAGE=${{ env.SRC_REPOSITORY }}/core:${{ env.SRC_TAG }} + DST_IMAGE=${{ env.DST_REPOSITORY }}/core:${{ env.DST_TAG }} + + echo "Pushing $SRC_IMAGE to $DST_IMAGE" + docker pull $SRC_IMAGE + docker tag $SRC_IMAGE $DST_IMAGE + docker push $DST_IMAGE + - name: Cross-push node + run: | + SRC_IMAGE=${{ env.SRC_REPOSITORY }}/core:${{ env.SRC_TAG }} + DST_IMAGE=${{ env.DST_REPOSITORY }}/core:${{ env.DST_TAG }} + + echo "Pushing $SRC_IMAGE to $DST_IMAGE" + docker pull $SRC_IMAGE + docker tag $SRC_IMAGE $DST_IMAGE + docker push $DST_IMAGE + + publish-docs: + if: github.event_name == 'release' + name: 📚 Publish Docs to GitHub Pages + runs-on: ubuntu-latest + needs: [build-core, docker-hub] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v2 + with: + name: core-documentation + path: .artifacts/core-documentation + + - uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install mkdocs theme & plugins + run: | + pip3 install --no-cache \ + 'mkdocs-git-revision-date-localized-plugin>=0.4' \ + 'mkdocs-material' \ + 'mkdocs-mermaid2-plugin' \ + 'mkdocs-codeinclude-plugin' \ + 'mkdocs-material-extensions' \ + 'mkdocs-simple-hooks' \ + 'git+http://github.com/TilBlechschmidt/mkdocs-helm' + + - name: Build & deploy documentation + env: + HELM_USE_GIT_TAG: true + run: | + mkdocs --version + mkdocs gh-deploy --force + + github-release: + if: github.event_name == 'release' + name: 🐙 Update GitHub Release + needs: [build-core, build-api] + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: core-executable + path: .artifacts/core-executable + + - uses: actions/download-artifact@v2 + with: + name: api-executable + path: .artifacts/api-executable + + - name: Build release asset upload url + run: | + RELEASE_ID=$(jq --raw-output '.release.id' $GITHUB_EVENT_PATH) + if [[ -z "${RELEASE_ID}" ]]; then + echo "There was no release ID in the GitHub event." + exit 1 + fi + + RELEASE_ASSET_UPLOAD_URL="https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets" + echo "$RELEASE_ASSET_UPLOAD_URL" + echo "RELEASE_ASSET_UPLOAD_URL=$RELEASE_ASSET_UPLOAD_URL" >> $GITHUB_ENV + + - name: Attach Core executable + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ env.RELEASE_ASSET_UPLOAD_URL }}?name=webgrid-core-linux + asset_path: .artifacts/core-executable/webgrid + asset_name: webgrid-core-linux + asset_content_type: application/octet-stream + + - name: Attach API executable + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ env.RELEASE_ASSET_UPLOAD_URL }}?name=webgrid-api.js + asset_path: .artifacts/api-executable/index.js + asset_name: webgrid-api.js + asset_content_type: application/javascript diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 1bb125b9..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,293 +0,0 @@ -name: 🚀 Project release - -on: - release: - types: [published] - -env: - CARGO_TERM_COLOR: always - -jobs: - build-core: - name: 🔨 Build Core component - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/cache@v2 - with: - path: core/.cache - key: build-core-${{ runner.os }}-lockfile=${{ hashFiles('**/Cargo.lock') }} - - - name: Build - run: make core - - - name: Change cache permission - run: sudo chmod -R 777 core/.cache - - - uses: actions/upload-artifact@v2 - with: - name: core-executable - path: .artifacts/core-executable/webgrid - - - uses: actions/upload-artifact@v2 - with: - name: core-documentation - path: .artifacts/core-documentation - - build-api: - name: 🔨 Build API component - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-node@v2-beta - with: - node-version: "14" - - - run: make api - - - uses: actions/upload-artifact@v2 - with: - name: api-executable - path: .artifacts/api-executable/index.js - - publish-api: - name: 🐳 Publish API to Docker Hub - runs-on: ubuntu-latest - needs: build-api - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: api-executable - path: .artifacts/api-executable - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=webgrid/api - VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [ "${{ github.event_name }}" = "push" ]; then - TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" - fi - echo "version=${VERSION}" >> $GITHUB_ENV - echo "tags=${TAGS}" >> $GITHUB_ENV - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: webgrid - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push webgrid/api Docker image - uses: docker/build-push-action@v2 - with: - context: . - file: distribution/docker/images/api/Dockerfile - tags: ${{ env.tags }} - push: ${{ github.event_name != 'pull_request' }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ env.created }} - org.opencontainers.image.revision=${{ github.sha }} - - publish-core: - name: 🐳 Publish Core to Docker Hub - runs-on: ubuntu-latest - needs: build-core - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: core-executable - path: .artifacts/core-executable - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=webgrid/core - VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [ "${{ github.event_name }}" = "push" ]; then - TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" - fi - echo "version=${VERSION}" >> $GITHUB_ENV - echo "tags=${TAGS}" >> $GITHUB_ENV - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: webgrid - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push webgrid/core Docker image - uses: docker/build-push-action@v2 - with: - context: . - file: distribution/docker/images/core/Dockerfile - tags: ${{ env.tags }} - push: ${{ github.event_name != 'pull_request' }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ env.created }} - org.opencontainers.image.revision=${{ github.sha }} - - publish-node: - name: 🐳 Publish Node to Docker Hub - runs-on: ubuntu-latest - needs: build-core - strategy: - matrix: - browser: ["chrome", "firefox"] - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: core-executable - path: .artifacts/core-executable - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=webgrid/node-${{ matrix.browser }} - VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [ "${{ github.event_name }}" = "push" ]; then - TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" - fi - echo "version=${VERSION}" >> $GITHUB_ENV - echo "tags=${TAGS}" >> $GITHUB_ENV - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: webgrid - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - - name: Build and push webgrid/node-${{ matrix.browser }} Docker image - uses: docker/build-push-action@v2 - with: - context: . - file: distribution/docker/images/node/Dockerfile - build-args: browser=${{ matrix.browser }} - tags: ${{ env.tags }} - push: ${{ github.event_name != 'pull_request' }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.created=${{ env.created }} - org.opencontainers.image.revision=${{ github.sha }} - - publish-docs: - name: 📚 Publish Docs to GitHub Pages - runs-on: ubuntu-latest - needs: build-core - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - uses: actions/download-artifact@v2 - with: - name: core-documentation - path: .artifacts/core-documentation - - - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Install mkdocs theme & plugins - run: | - pip3 install --no-cache \ - 'mkdocs-git-revision-date-localized-plugin>=0.4' \ - 'mkdocs-material' \ - 'mkdocs-mermaid2-plugin' \ - 'mkdocs-codeinclude-plugin' \ - 'mkdocs-material-extensions' \ - 'mkdocs-simple-hooks' \ - 'git+http://github.com/TilBlechschmidt/mkdocs-helm' - - - name: Build & deploy documentation - env: - HELM_USE_GIT_TAG: true - run: | - mkdocs --version - mkdocs gh-deploy --force - - github-release: - name: 🐙 Update GitHub Release - needs: [build-core, build-api] - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: core-executable - path: .artifacts/core-executable - - - uses: actions/download-artifact@v2 - with: - name: api-executable - path: .artifacts/api-executable - - - name: Build release asset upload url - run: | - RELEASE_ID=$(jq --raw-output '.release.id' $GITHUB_EVENT_PATH) - if [[ -z "${RELEASE_ID}" ]]; then - echo "There was no release ID in the GitHub event." - exit 1 - fi - - RELEASE_ASSET_UPLOAD_URL="https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets" - echo "$RELEASE_ASSET_UPLOAD_URL" - echo "RELEASE_ASSET_UPLOAD_URL=$RELEASE_ASSET_UPLOAD_URL" >> $GITHUB_ENV - - - name: Attach Core executable - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ env.RELEASE_ASSET_UPLOAD_URL }}?name=webgrid-core-linux - asset_path: .artifacts/core-executable/webgrid - asset_name: webgrid-core-linux - asset_content_type: application/octet-stream - - - name: Attach API executable - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ env.RELEASE_ASSET_UPLOAD_URL }}?name=webgrid-api.js - asset_path: .artifacts/api-executable/index.js - asset_name: webgrid-api.js - asset_content_type: application/javascript