From a1a026383fac5d6e3dc6908b802685208e1cf1d8 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 18 Dec 2024 08:21:10 -0800 Subject: [PATCH] ci: Restructure release workflow (#60) This isolates elevated permissions to the release publication job only, and simplifies a more complex sequence of creating a draft release, then building and attaching binaries, then compiling release notes, then publishing the release. Now we simply build, compile notes, then publish a full release with notes and binaries at once. This also removes the need for our own "api client" in JavaScript. Now we perform these actions with GitHub's own tools: "gh" command line to create the release and "actions/" official actions to upload and download build artifacts. --- .github/workflows/build.yaml | 25 ---- .github/workflows/release.yaml | 82 +++-------- api-client/.gitignore | 1 - api-client/main.js | 248 --------------------------------- api-client/package-lock.json | 144 ------------------- api-client/package.json | 5 - 6 files changed, 20 insertions(+), 485 deletions(-) delete mode 100644 api-client/.gitignore delete mode 100644 api-client/main.js delete mode 100644 api-client/package-lock.json delete mode 100644 api-client/package.json diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a227547..2f78146 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,17 +22,9 @@ on: # workflows. workflow_call: inputs: - release_id: - required: false - type: string ref: required: true type: string - secrets: - # The GITHUB_TOKEN name is reserved, but not passed through implicitly. - # So we call our secret parameter simply TOKEN. - TOKEN: - required: false # Runs on manual trigger. workflow_dispatch: @@ -214,23 +206,6 @@ jobs: - name: Check that executables are static run: ./repo-src/build-scripts/99-check-static.sh - - name: Attach assets to release - if: inputs.release_id != '' - env: - GITHUB_TOKEN: ${{ secrets.TOKEN }} - run: | - set -e - set -x - - # Attach the build outputs to the draft release. Each machine will - # do this separately and in parallel. Later, another job will take - # over to collect them all and use their MD5 sums to create the - # release notes (the "body" of the release). - release_id="${{ inputs.release_id }}" - (cd ./repo-src/api-client && npm ci) - node ./repo-src/api-client/main.js \ - upload-all-assets "$release_id" assets/ - - name: Debug uses: mxschmitt/action-tmate@v3.6 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6274039..8caf83d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -30,54 +30,29 @@ on: # will have to opt in after setting up their own self-hosted runners. jobs: - # On a single Linux host, draft a release. Later, different hosts will build - # for each OS/CPU in parallel, and then attach the resulting binaries to this - # draft. - draft_release: - name: Draft release - runs-on: ubuntu-latest - outputs: - release_id: ${{ steps.draft_release.outputs.release_id }} - steps: - - uses: actions/checkout@v4 - with: - path: repo-src - ref: ${{ github.ref }} - - - name: Draft release - id: draft_release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -e - set -x - - # Create a draft release associated with the tag that triggered this - # workflow. - tag="${{ github.ref }}" - (cd repo-src/api-client && npm ci) - release_id=$(node ./repo-src/api-client/main.js draft-release "$tag") - echo "::set-output name=release_id::$release_id" - build: - needs: draft_release uses: ./.github/workflows/build.yaml with: - release_id: ${{ needs.draft_release.outputs.release_id }} ref: ${{ github.ref }} - secrets: - TOKEN: ${{ secrets.GITHUB_TOKEN }} publish_release: name: Publish release - needs: [draft_release, build] + needs: [build] runs-on: ubuntu-latest + permissions: + # "Write" to contents is necessary to create a release. + contents: write steps: - uses: actions/checkout@v4 with: path: repo-src ref: ${{ github.ref }} + - uses: actions/download-artifact@v4 + with: + path: assets + merge-multiple: true + - name: Publish release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -93,8 +68,8 @@ jobs: echo " - $(date -I)" >> body.txt echo "" >> body.txt - echo "$GITHUB_REPOSITORY version:" >> body.txt - echo " - $repo_tag" >> body.txt + echo "${{ github.repository }} version:" >> body.txt + echo " - ${{ github.ref_name }}" >> body.txt echo "" >> body.txt echo "Software versions:" >> body.txt @@ -102,32 +77,15 @@ jobs: sed -e 's/^/ - /' >> body.txt echo "" >> body.txt - # Update the release notes with this preliminary version. This is - # what gets emailed out when we publish the release below. - release_id="${{ needs.draft_release.outputs.release_id }}" - (cd repo-src/api-client && npm ci) - node ./repo-src/api-client/main.js \ - update-release-body "$release_id" "$(cat body.txt)" - - # Now we have to take the release out of draft mode. Until we do, we - # can't get download URLs for the assets. - node ./repo-src/api-client/main.js \ - publish-release "$release_id" - - # The downloads are sometimes a bit flaky (responding with 404) if we - # don't put some delay between publication and download. This number - # is arbitrary, but experimentally, it seems to solve the issue. - sleep 30 - - # Next, download the assets. - node ./repo-src/api-client/main.js \ - download-all-assets "$release_id" assets/ - - # Now add the MD5 sums to the release notes. + # Add the MD5 sums to the release notes. echo "MD5 sums:" >> body.txt (cd assets; md5sum * | sed -e 's/^/ - /') >> body.txt - # Now update the release notes one last time, with the MD5 sums - # appended. - node ./repo-src/api-client/main.js \ - update-release-body "$release_id" "$(cat body.txt)" + # Publish the release, including release notes and assets. + gh release create \ + -R ${{ github.repository }} \ + --verify-tag \ + --notes-file body.txt \ + --title "${{ github.ref_name }}" \ + "${{ github.ref_name }}" \ + assets/* diff --git a/api-client/.gitignore b/api-client/.gitignore deleted file mode 100644 index 3c3629e..0000000 --- a/api-client/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/api-client/main.js b/api-client/main.js deleted file mode 100644 index 93c15d5..0000000 --- a/api-client/main.js +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// A script to communicate with the GitHub API to perform certain actions in -// the workflow. - -const fs = require('fs'); -const https = require('https'); -const path = require('path'); - -// octokit is the official API client of GitHub. -const { Octokit } = require('@octokit/core'); - -const repo = process.env['GITHUB_REPOSITORY']; - -const octokit = new Octokit({ - auth: process.env['GITHUB_TOKEN'], -}); - -const COMMAND_MAP = {}; - -const MAX_REDIRECTS = 3; - - -// Convert a camelCase name to kebab-case. -function camelCaseToKebabCase(name) { - // Split the camelCase name into parts with a zero-length lookahead regex on - // any capital letter. Something like "methodName" should be split into - // ["method", "Name"]. - const nameParts = name.split(/(?=[A-Z])/); - - // Convert those parts into a kebab-case name. - return nameParts.map(part => part.toLowerCase()).join('-'); -} - -// Register a method that the user can invoke on the command-line. We use -// (cheap) introspection to find the argument names, so that we can -// automatically document usage of each command without worrying about the docs -// getting out of sync with the code. -function registerCommand(method) { - const methodName = method.name; - const commandName = camelCaseToKebabCase(methodName); - - // Hack out the arguments from the stringified function. This is terrible - // and will not work in the general case of all JavaScript, but it does work - // here. (Don't be like me.) - const firstLine = method.toString().split('\n')[0]; - const argString = firstLine.split('(')[1].split(')')[0]; - const camelArgs = argString.replace(/\s+/g, '').split(','); - const args = camelArgs.map(camelCaseToKebabCase); - - COMMAND_MAP[commandName] = { - commandName, - method, - args, - }; -} - -// A helper function to make calls to the GitHub Repo API. -async function repoApiCall(method, apiPath, data, upload=false) { - const url = `${method} /repos/${repo}${apiPath}`; - - // Clone the "data" passed in. - const options = Object.assign({}, data); - - // If we're uploading, that goes to a different API endpoint. - if (upload) { - options.baseUrl = 'https://uploads.github.com'; - } - - const response = await octokit.request(url, options); - return response.data; -} - - -async function draftRelease(tagName) { - // Turns "refs/tags/foo" into "foo". - tagName = tagName.replace('refs/tags/', ''); - - const response = await repoApiCall('POST', '/releases', { - tag_name: tagName, - name: tagName, - draft: true, - }); - - return response.id; -} -registerCommand(draftRelease); - -async function uploadAsset(releaseId, assetPath) { - const baseName = path.basename(assetPath); - const data = await fs.promises.readFile(assetPath); - - const apiPath = `/releases/${releaseId}/assets?name=${baseName}`; - await repoApiCall('POST', apiPath, { - headers: { - 'content-type': 'application/octet-stream', - 'content-length': data.length, - }, - data, - }, /* upload= */ true); -} -// Not registered as an independent command. - -async function uploadAllAssets(releaseId, folderPath) { - const folderContents = await fs.promises.readdir(folderPath); - for (const assetFilename of folderContents) { - const assetPath = path.join(folderPath, assetFilename); - await uploadAsset(releaseId, assetPath); - } -} -registerCommand(uploadAllAssets); - -// A helper function that will fetch via HTTPS and follow redirects. -function fetchViaHttps(url, outputStream, redirectCount=0) { - if (redirectCount > MAX_REDIRECTS) { - throw new Error('Too many redirects!'); - } - - return new Promise((resolve, reject) => { - const request = https.get(url, (response) => { - if (response.statusCode == 301 || response.statusCode == 302) { - // Handle HTTP redirects. - const newUrl = response.headers.location; - - resolve(fetchViaHttps(newUrl, outputStream, redirectCount + 1)); - } else if (response.statusCode == 200) { - response.pipe(outputStream); - outputStream.on('finish', resolve); - } else { - reject(new Error(`Bad HTTP status code: ${response.statusCode}`)); - } - }); - request.on('error', reject); - }); -} - -async function downloadAllAssets(releaseId, outputPath) { - // If the output path does not exist, create it. - try { - await fs.promises.stat(outputPath); - } catch (error) { - await fs.promises.mkdir(outputPath); - } - - const apiPath = `/releases/${releaseId}/assets`; - const assetList = await repoApiCall('GET', apiPath); - for (const asset of assetList) { - const url = asset.browser_download_url; - const assetPath = path.join(outputPath, asset.name); - const outputStream = fs.createWriteStream(assetPath); - - console.log(`Fetching ${url} to ${assetPath}`); - await fetchViaHttps(url, outputStream); - } -} -registerCommand(downloadAllAssets); - -async function publishRelease(releaseId) { - await repoApiCall('PATCH', `/releases/${releaseId}`, { draft: false }); -} -registerCommand(publishRelease); - -async function updateReleaseBody(releaseId, body) { - // NOTE: If you update the release body without specifying tag_name, it gets - // reset, resulting in a new tag being created with an auto-generated name - // like "untagged-SHA1". So we need to fetch the existing name before we - // update the body, and we need to specify it here. This is not mentioned in - // GitHub's docs, and may be a bug on their end. - const release = await getRelease(releaseId); - await repoApiCall('PATCH', `/releases/${releaseId}`, { - body, - tag_name: release.tag_name, - }); -} -registerCommand(updateReleaseBody); - -async function getRelease(releaseId) { - return await repoApiCall('GET', `/releases/${releaseId}`); -} -registerCommand(getRelease); - - -// We expect a command and arguments. -const commandName = process.argv[2]; -const args = process.argv.slice(3); -const command = COMMAND_MAP[commandName]; -let okay = true; - -if (!commandName) { - console.error('No command selected!'); - okay = false; -} else if (!command) { - console.error(`Unknown command: ${commandName}`); - okay = false; -} else if (args.length != command.args.length) { - console.error(`Wrong number of arguments for command: ${commandName}`); - okay = false; -} - -// If something is wrong with the way the script was called, print usage -// information. The list of commands and their arguments are gleaned from -// COMMAND_MAP, which was populated by registerCommand() and introspection of -// the command functions. -if (!okay) { - console.error(''); - console.error('Usage:'); - const thisScript = path.basename(process.argv[1]); - - for (possibleCommand of Object.values(COMMAND_MAP)) { - console.error( - ' ', - thisScript, - possibleCommand.commandName, - ...possibleCommand.args.map(arg => `<${arg}>`)); - } - process.exit(1); -} - -// Run the command with the given arguments. -(async () => { - let response; - - try { - response = await command.method(...args); - } catch (error) { - console.error('Command failed!'); - console.error(''); - console.error(error); - process.exit(1); - } - - // If there's a return value, print it. - if (response) { - console.log(response); - } -})(); diff --git a/api-client/package-lock.json b/api-client/package-lock.json deleted file mode 100644 index 40c88d6..0000000 --- a/api-client/package-lock.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "requires": true, - "lockfileVersion": 1, - "dependencies": { - "@octokit/auth-token": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.5.tgz", - "integrity": "sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA==", - "requires": { - "@octokit/types": "^6.0.3" - } - }, - "@octokit/core": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.5.1.tgz", - "integrity": "sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw==", - "requires": { - "@octokit/auth-token": "^2.4.4", - "@octokit/graphql": "^4.5.8", - "@octokit/request": "^5.6.0", - "@octokit/request-error": "^2.0.5", - "@octokit/types": "^6.0.3", - "before-after-hook": "^2.2.0", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/endpoint": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", - "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", - "requires": { - "@octokit/types": "^6.0.3", - "is-plain-object": "^5.0.0", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/graphql": { - "version": "4.6.4", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.6.4.tgz", - "integrity": "sha512-SWTdXsVheRmlotWNjKzPOb6Js6tjSqA2a8z9+glDJng0Aqjzti8MEWOtuT8ZSu6wHnci7LZNuarE87+WJBG4vg==", - "requires": { - "@octokit/request": "^5.6.0", - "@octokit/types": "^6.0.3", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/openapi-types": { - "version": "9.6.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-9.6.0.tgz", - "integrity": "sha512-L+8x7DpcNtHkMbTxxCxg3cozvHUNP46rOIzFwoMs0piWwQzAGNXqlIQO2GLvnKTWLUh99DkY+UyHVrP4jXlowg==" - }, - "@octokit/request": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.0.tgz", - "integrity": "sha512-4cPp/N+NqmaGQwbh3vUsYqokQIzt7VjsgTYVXiwpUP2pxd5YiZB2XuTedbb0SPtv9XS7nzAKjAuQxmY8/aZkiA==", - "requires": { - "@octokit/endpoint": "^6.0.1", - "@octokit/request-error": "^2.1.0", - "@octokit/types": "^6.16.1", - "is-plain-object": "^5.0.0", - "node-fetch": "^2.6.1", - "universal-user-agent": "^6.0.0" - } - }, - "@octokit/request-error": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", - "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", - "requires": { - "@octokit/types": "^6.0.3", - "deprecation": "^2.0.0", - "once": "^1.4.0" - } - }, - "@octokit/types": { - "version": "6.25.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.25.0.tgz", - "integrity": "sha512-bNvyQKfngvAd/08COlYIN54nRgxskmejgywodizQNyiKoXmWRAjKup2/LYwm+T9V0gsKH6tuld1gM0PzmOiB4Q==", - "requires": { - "@octokit/openapi-types": "^9.5.0" - } - }, - "before-after-hook": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", - "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" - }, - "deprecation": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", - "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" - }, - "is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" - }, - "node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - } - } -} diff --git a/api-client/package.json b/api-client/package.json deleted file mode 100644 index 94e3904..0000000 --- a/api-client/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "dependencies": { - "@octokit/core": "^3.5.1" - } -}