From ab5d76f0300860f0a5935442c9c6ed98c92524a9 Mon Sep 17 00:00:00 2001 From: Paul Jolly Date: Thu, 27 Jul 2023 13:51:45 +0100 Subject: [PATCH] internal/ci: split target branch workflows by repo We only want to include macOS in the build matrix for a target branch in the main repo. Running it in the trybot repo is a waste of time and resources because it will never be used. The only way to achieve this sensibly is to template the trybot_targetBranch workflow and make it conditional on the repo in which the workflow is running. Also fix the running of pull request workflows, which was inadvertently broken in a previous change (and can only be tested after a merge to alpha). Signed-off-by: Paul Jolly Change-Id: Ifbf3eeea16853195e12e2821a970bf8089af32cc --- .github/workflows/trybot.yml | 32 ++- ...Branch.yml => trybot_githubRepository.yml} | 32 +-- internal/ci/github/trybot.cue | 259 +++++++++--------- internal/ci/github/workflows.cue | 6 +- 4 files changed, 163 insertions(+), 166 deletions(-) rename .github/workflows/{trybot_targetBranch.yml => trybot_githubRepository.yml} (98%) diff --git a/.github/workflows/trybot.yml b/.github/workflows/trybot.yml index 899c683fb1..aa1860a781 100644 --- a/.github/workflows/trybot.yml +++ b/.github/workflows/trybot.yml @@ -2,16 +2,32 @@ name: TryBot "on": + pull_request: {} push: branches: + - ci/test - master - alpha tags-ignore: - v* - pull_request: {} workflow_dispatch: {} jobs: test: + strategy: + fail-fast: false + matrix: + go-version: + - 1.20.x + runner: + - ubuntu-22.04 + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: bash + if: |- + (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"trybot"')) || github.event_name = 'pull_request' || (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"')) && (github.repository == 'cue-lang/cuelang.org-trybot') steps: - if: runner.os == 'macOS' name: Update Homebrew (macOS) @@ -227,17 +243,3 @@ jobs: ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }} ALGOLIA_INDEX_NAME: cuelang.org ALGOLIA_INDEX_FILE: ../_public/algolia.json - if: |- - (contains(github.event.head_commit.message, ' - Dispatch-Trailer: {"type":"trybot"')) - strategy: - fail-fast: false - matrix: - go-version: - - 1.20.x - runner: - - ubuntu-22.04 - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: bash diff --git a/.github/workflows/trybot_targetBranch.yml b/.github/workflows/trybot_githubRepository.yml similarity index 98% rename from .github/workflows/trybot_targetBranch.yml rename to .github/workflows/trybot_githubRepository.yml index 573db1b042..7682277ef2 100644 --- a/.github/workflows/trybot_targetBranch.yml +++ b/.github/workflows/trybot_githubRepository.yml @@ -1,6 +1,6 @@ # Code generated internal/ci/ci_tool.cue; DO NOT EDIT. -name: TryBot target branch +name: TryBot (cue-lang/cuelang.org target branch "on": push: branches: @@ -11,6 +11,21 @@ name: TryBot target branch - v* jobs: test: + strategy: + fail-fast: false + matrix: + go-version: + - 1.20.x + runner: + - ubuntu-22.04 + - macos-13 + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: bash + if: |- + (! (contains(github.event.head_commit.message, ' + Dispatch-Trailer: {"type":"'))) && (github.repository == 'cue-lang/cuelang.org') steps: - if: runner.os == 'macOS' name: Update Homebrew (macOS) @@ -226,18 +241,3 @@ jobs: ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_INDEX_KEY }} ALGOLIA_INDEX_NAME: cuelang.org ALGOLIA_INDEX_FILE: ../_public/algolia.json - if: |- - ! (contains(github.event.head_commit.message, ' - Dispatch-Trailer: {"type":"')) - strategy: - fail-fast: false - matrix: - go-version: - - 1.20.x - runner: - - ubuntu-22.04 - - macos-13 - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: bash diff --git a/internal/ci/github/trybot.cue b/internal/ci/github/trybot.cue index 59625deaf0..cb037feef7 100644 --- a/internal/ci/github/trybot.cue +++ b/internal/ci/github/trybot.cue @@ -24,33 +24,38 @@ import ( "github.com/cue-lang/cuelang.org/internal/ci/netlify" ) +// The trybot workflow runs for: +// +// * CLs (in the trybot repo) and PRs +// * the target branch in the trybot repo +// +// In all of the situations above, we only want to run with Linux in the build +// matrix because the macOS runner is very slow. workflows: trybot: _trybot & { name: _repo.trybot.name - on: { - push: { - branches: _repo.protectedBranchPatterns // do not run PR branches, or the test default branch. - "tags-ignore": [_repo.releaseTagPattern] - } - pull_request: {} - } + on: pull_request: {} jobs: test: { - if: "\(_repo.containsTrybotTrailer)" - strategy: { - "fail-fast": false - matrix: { - "go-version": [_repo.latestStableGo] - runner: [_repo.linuxMachine] - } - } - "runs-on": "${{ matrix.runner }}" + if: "\(_repo.containsTrybotTrailer) || github.event_name = 'pull_request' || (! \(_repo.containsDispatchTrailer) && (github.repository == '\(_repo.trybotRepositoryPath)')" + strategy: matrix: runner: [_repo.linuxMachine] } } -workflows: trybot_targetBranch: _trybot & { - name: _repo.trybot.name + " target branch" +// The trybot_githubRepository workflow runs for the target branch in the main +// repo. In this situation (and only this situation), we want to run with Linux +// and macOS in the build matrix. +workflows: trybot_githubRepository: _trybot & { + name: _repo.trybot.name + " (\(_repo.githubRepositoryPath) target branch" + jobs: test: { + if: "(! \(_repo.containsDispatchTrailer)) && (github.repository == '\(_repo.githubRepositoryPath)')" + strategy: matrix: runner: [_repo.linuxMachine, _repo.macosMachine] + } +} + +// The trybot workflow. +_trybot: _repo.bashWorkflow & { on: { push: { branches: list.Concat([[_repo.testDefaultBranch], _repo.protectedBranchPatterns]) // do not run PR branches @@ -59,128 +64,118 @@ workflows: trybot_targetBranch: _trybot & { } jobs: test: { - if: "! \(_repo.containsDispatchTrailer)" strategy: { "fail-fast": false matrix: { "go-version": [_repo.latestStableGo] - runner: [_repo.linuxMachine, _repo.macosMachine] } } "runs-on": "${{ matrix.runner }}" - } - -} -// The trybot workflow. -_trybot: _repo.bashWorkflow & { - jobs: { - test: { - steps: [ - _updateHomebrew, - - for v in _repo.checkoutCode {v}, - - _repo.earlyChecks, - - for v in _installDockerMacOS {v}, - - _installMacOSUtils, - _setupBuildx, - _installNode, - _installGo, - _installHugoLinux, - _installHugoMacOS, - - // cachePre must come after installing Node and Go, because the cache locations - // are established by running each tool. - for v in _setupGoActionsCaches {v}, - - // Disable checkout for "latest" CUE for now. Go does not (yet) - // handle a query for cuelang.org/go@v0.6 when there is only a - // prerelease version matching that query. - // - // json.#step & { - // // The latest git clean check ensures that this call is effectively - // // side effect-free. Using GOPRIVATE ensures we don't accidentally - // // hit a stale cache in the proxy. - // name: "Ensure latest CUE" - // run: """ - // GOPRIVATE=cuelang.org/go go get -d cuelang.org/go@latest - // go mod tidy - // go mod tidy - // """ - // }, - - // Rebuild docker image - json.#step & { - run: "./_scripts/buildDockerImage.bash" - }, - - // Go generate steps - _goGenerate & { - name: "Regenerate" - }, - - // npm install in hugo to allow serve test to pass - // - // TODO: make this a more principled change. - json.#step & { - run: "npm install" - "working-directory": "hugo" - }, - - // Go test steps - _goTest & { - name: "Test" - }, - - // Run staticcheck - json.#step & { - name: "staticcheck" - run: "./_scripts/staticcheck.bash" - }, - - // go mod tidy - _modTidy & { - name: "Check module is tidy" - }, - - _dist, - _repo.checkGitClean, - - // Now the frontend build has happened, ensure that linters pass - json.#step & { - "working-directory": "hugo" - run: """ - npm run lint - """ - }, - - // Only run a deploy of tip if we are running as part of the trybot repo, - // with a TryBot-Trailer, i.e. as part of CI check of the trybot workflow - _netlifyDeploy & { - if: "github.repository == '\(_repo.trybotRepositoryPath)' && \(_repo.containsTrybotTrailer) && \(_isLatestLinux)" - #site: _repo.netlifySites.cls - #alias: "cl-${{ \(_dispatchTrailerExpr).CL }}-${{ \(_dispatchTrailerExpr).patchset }}" - name: "Deploy preview of CL" - }, - - json.#step & { - // Only run in the main repo on the alpha branch. Because anywhere else - // doesn't make sense. - if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.alphaBranch)') && \(_isLatestLinux)" - run: "npm run algolia" - "working-directory": "hugo" - env: { - ALGOLIA_APP_ID: "5LXFM0O81Q" - ALGOLIA_ADMIN_KEY: "${{ secrets.ALGOLIA_INDEX_KEY }}" - ALGOLIA_INDEX_NAME: "cuelang.org" - ALGOLIA_INDEX_FILE: "../_public/algolia.json" - } - }, - ] - } + steps: [ + _updateHomebrew, + + for v in _repo.checkoutCode {v}, + + _repo.earlyChecks, + + for v in _installDockerMacOS {v}, + + _installMacOSUtils, + _setupBuildx, + _installNode, + _installGo, + _installHugoLinux, + _installHugoMacOS, + + // cachePre must come after installing Node and Go, because the cache locations + // are established by running each tool. + for v in _setupGoActionsCaches {v}, + + // Disable checkout for "latest" CUE for now. Go does not (yet) + // handle a query for cuelang.org/go@v0.6 when there is only a + // prerelease version matching that query. + // + // json.#step & { + // // The latest git clean check ensures that this call is effectively + // // side effect-free. Using GOPRIVATE ensures we don't accidentally + // // hit a stale cache in the proxy. + // name: "Ensure latest CUE" + // run: """ + // GOPRIVATE=cuelang.org/go go get -d cuelang.org/go@latest + // go mod tidy + // go mod tidy + // """ + // }, + + // Rebuild docker image + json.#step & { + run: "./_scripts/buildDockerImage.bash" + }, + + // Go generate steps + _goGenerate & { + name: "Regenerate" + }, + + // npm install in hugo to allow serve test to pass + // + // TODO: make this a more principled change. + json.#step & { + run: "npm install" + "working-directory": "hugo" + }, + + // Go test steps + _goTest & { + name: "Test" + }, + + // Run staticcheck + json.#step & { + name: "staticcheck" + run: "./_scripts/staticcheck.bash" + }, + + // go mod tidy + _modTidy & { + name: "Check module is tidy" + }, + + _dist, + _repo.checkGitClean, + + // Now the frontend build has happened, ensure that linters pass + json.#step & { + "working-directory": "hugo" + run: """ + npm run lint + """ + }, + + // Only run a deploy of tip if we are running as part of the trybot repo, + // with a TryBot-Trailer, i.e. as part of CI check of the trybot workflow + _netlifyDeploy & { + if: "github.repository == '\(_repo.trybotRepositoryPath)' && \(_repo.containsTrybotTrailer) && \(_isLatestLinux)" + #site: _repo.netlifySites.cls + #alias: "cl-${{ \(_dispatchTrailerExpr).CL }}-${{ \(_dispatchTrailerExpr).patchset }}" + name: "Deploy preview of CL" + }, + + json.#step & { + // Only run in the main repo on the alpha branch. Because anywhere else + // doesn't make sense. + if: "github.repository == '\(_repo.githubRepositoryPath)' && (github.ref == 'refs/heads/\(_repo.alphaBranch)') && \(_isLatestLinux)" + run: "npm run algolia" + "working-directory": "hugo" + env: { + ALGOLIA_APP_ID: "5LXFM0O81Q" + ALGOLIA_ADMIN_KEY: "${{ secrets.ALGOLIA_INDEX_KEY }}" + ALGOLIA_INDEX_NAME: "cuelang.org" + ALGOLIA_INDEX_FILE: "../_public/algolia.json" + } + }, + ] } let matrixRunner = "matrix.runner" diff --git a/internal/ci/github/workflows.cue b/internal/ci/github/workflows.cue index db7cdc2d44..4a298764c4 100644 --- a/internal/ci/github/workflows.cue +++ b/internal/ci/github/workflows.cue @@ -41,9 +41,9 @@ workflows: close({ _repo.trybotWorkflows - trybot: _ - trybot_targetBranch: _ - update_tip: _linuxWorkflow + trybot: _ + trybot_githubRepository: _ + update_tip: _linuxWorkflow trybot_dispatch: #dummyDispatch: dummyDispatch })