diff --git a/.github/actions/equivalence-test/action.yml b/.github/actions/equivalence-test/action.yml index 7f2eebf77772..2763f028f1a5 100644 --- a/.github/actions/equivalence-test/action.yml +++ b/.github/actions/equivalence-test/action.yml @@ -2,14 +2,8 @@ # SPDX-License-Identifier: BUSL-1.1 name: equivalence-test -description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests" +description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests and update the golden files." inputs: - target-terraform-version: - description: "The version of Terraform to use in execution." - required: true - target-terraform-branch: - description: "The branch within this repository to update and compare." - required: true target-equivalence-test-version: description: "The version of the Terraform equivalence tests to use." default: "0.3.0" @@ -19,9 +13,22 @@ inputs: target-arch: description: "Current architecture" default: "amd64" + current-branch: + description: "What branch are we currently on?" + required: true + new-branch: + description: "Name of new branch to be created for the review." + required: true + reviewers: + description: "Comma-separated list of GitHub usernames to request review from." + required: true + message: + description: "Message to include in the commit." + required: true runs: using: "composite" steps: + - name: "download equivalence test binary" shell: bash run: | @@ -30,15 +37,13 @@ runs: ./bin/equivalence-tests \ ${{ inputs.target-os }} \ ${{ inputs.target-arch }} - - name: "download terraform binary" + + - name: Build terraform shell: bash - run: | - ./.github/scripts/equivalence-test.sh download_terraform_binary \ - ${{ inputs.target-terraform-version }} \ - ./bin/terraform \ - ${{ inputs.target-os }} \ - ${{ inputs.target-arch }} + run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform + - name: "run and update equivalence tests" + id: execute shell: bash run: | ./bin/equivalence-tests update \ @@ -47,15 +52,24 @@ runs: --binary=$(pwd)/bin/terraform changed=$(git diff --quiet -- testing/equivalence-tests/outputs || echo true) - if [[ $changed == "true" ]]; then - echo "found changes, and pushing new golden files into branch ${{ inputs.target-terraform-branch }}." - - git config user.email "52939924+teamterraform@users.noreply.github.com" - git config user.name "The Terraform Team" + echo "changed=$changed" >> "${GITHUB_OUTPUT}" - git add ./testing/equivalence-tests/outputs - git commit -m "Automated equivalence test golden file update for release ${{ inputs.target-terraform-version }}." - git push - else - echo "found no changes, so not pushing any updates." - fi + - name: "branch, commit, and push changes" + if: steps.execute.outputs.changed == 'true' + shell: bash + run: | + git checkout -b ${{ inputs.new-branch }} + git add testing/equivalence-tests/outputs + git commit -m "Update equivalence test golden files." + git push --set-upstream origin ${{ inputs.new-branch }} + + - name: "create pull request" + if: steps.execute.outputs.changed == 'true' + shell: bash + run: | + gh pr create \ + --base ${{ inputs.current-branch }} \ + --head ${{ inputs.new-branch }} \ + --title "Update equivalence test golden files" \ + --body "This PR updates the equivalence test golden files." \ + --reviewer ${{ inputs.reviewers }} diff --git a/.github/scripts/equivalence-test.sh b/.github/scripts/equivalence-test.sh index e3af36993318..948c107c5623 100755 --- a/.github/scripts/equivalence-test.sh +++ b/.github/scripts/equivalence-test.sh @@ -26,11 +26,11 @@ Commands: ./equivalence-test.sh download_equivalence_test_binary 0.3.0 ./bin/terraform-equivalence-testing linux amd64 - download_terraform_binary - download_terraform_binary downloads the terraform release binary for a given - version and places it at the target path. + build_terraform_binary + download_terraform_binary builds the Terraform binary and places it at the + target path. - ./equivalence-test.sh download_terraform_binary 1.4.3 ./bin/terraform linux amd64 + ./equivalence-test.sh build_terraform_binary ./bin/terraform EOF } @@ -65,25 +65,17 @@ function download_equivalence_test_binary { rm releases.json } -function download_terraform_binary { - VERSION="${1:-}" - TARGET="${2:-}" - OS="${3:-}" - ARCH="${4:-}" +function build_terraform_binary { + TARGET="${1:-}" - if [[ -z "$VERSION" || -z "$TARGET" || -z "$OS" || -z "$ARCH" ]]; then - echo "missing at least one of [, , , ] arguments" + if [[ -z "$TARGET" ]]; then + echo "target argument" usage exit 1 fi - mkdir -p zip - curl "https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_${OS}_${ARCH}.zip" > "zip/terraform.zip" - - mkdir -p bin - unzip -p "zip/terraform.zip" terraform > "$TARGET" + go build -o "$TARGET" ./ chmod u+x "$TARGET" - rm -r zip } function get_target_branch { @@ -142,14 +134,14 @@ function main { download_equivalence_test_binary "$2" "$3" "$4" "$5" ;; - download_terraform_binary) - if [ "${#@}" != 5 ]; then + build_terraform_binary) + if [ "${#@}" != 2 ]; then echo "invalid number of arguments" usage exit 1 fi - download_terraform_binary "$2" "$3" "$4" "$5" + build_terraform_binary "$2" ;; *) diff --git a/.github/workflows/crt-hook-equivalence-tests.yml b/.github/workflows/crt-hook-equivalence-tests.yml deleted file mode 100644 index e37cde5efa8d..000000000000 --- a/.github/workflows/crt-hook-equivalence-tests.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: crt-hook-equivalence-tests - -on: - repository_dispatch: - types: - - crt-hook-equivalence-tests::terraform::* - -permissions: - contents: write - -jobs: - parse-metadata: - name: "Parse metadata.json" - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - target-branch: ${{ steps.parse.outputs.target-branch }} - steps: - - name: parse - id: parse - env: - METADATA_PAYLOAD: ${{ toJSON(github.event.client_payload.payload) }} - run: | - VERSION=$(echo ${METADATA_PAYLOAD} | jq -r '.version') - TARGET_BRANCH=$(./.github/scripts/equivalence-test.sh get-target-branch "$VERSION") - - echo "target-branch=$TARGET_BRANCH" >> "GITHUB_OUTPUT" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - run-equivalence-tests: - runs-on: ubuntu-latest - name: "Run equivalence tests" - needs: - - parse-metadata - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - ref: ${{ needs.parse-metadata.outputs.target-branch }} - - uses: ./.github/actions/equivalence-test - with: - target-terraform-version: ${{ needs.parse-metadata.outputs.version }} - target-terraform-branch: ${{ needs.parse-metadata.outputs.target-branch }} - target-equivalence-test-version: 0.3.0 - target-os: linux - target-arch: amd64 diff --git a/.github/workflows/equivalence-test-diff.yml b/.github/workflows/equivalence-test-diff.yml new file mode 100644 index 000000000000..6967cab5c41d --- /dev/null +++ b/.github/workflows/equivalence-test-diff.yml @@ -0,0 +1,70 @@ +name: equivalence-test-diff + +on: + pull_request: + types: + - synchronize + - ready_for_review + - reopened + - synchronize + +permissions: + contents: read + pull-requests: write + +jobs: + equivalence-test-diff: + name: "Equivalence Test Diff" + runs-on: ubuntu-latest + + steps: + - name: Fetch source code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Determine Go version + id: go + uses: ./.github/actions/go-version + + - name: Install Go toolchain + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + with: + go-version: ${{ steps.go.outputs.version }} + cache-dependency-path: go.sum + + - name: Download testing framework + shell: bash + run: | + ./.github/scripts/equivalence-test.sh download_equivalence_test_binary \ + 0.3.0 \ + ./bin/equivalence-tests \ + linux \ + amd64 + + - name: Build terraform + shell: bash + run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform + + - name: Run equivalence tests + id: equivalence-tests + shell: bash {0} # we want to capture the exit code + run: | + ./bin/equivalence-tests diff \ + --tests=testing/equivalence-tests/tests \ + --goldens=testing/equivalence-tests/outputs \ + --binary=$(pwd)/bin/terraform + echo "exit-code=$?" >> "${GITHUB_OUTPUT}" + + - name: Equivalence tests failed + if: steps.equivalence-tests.outputs.exit-code == 1 # 1 is the exit code for failure + shell: bash + run: | + gh pr comment ${{ github.event.pull_request.number }} \ + --body "The equivalence tests failed. Please investigate [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})." + exit 1 # fail the job + + - name: Equivalence tests changed + if: steps.equivalence-tests.outputs.exit-code == 2 # 2 is the exit code for changed + shell: bash + run: | + gh pr comment ${{ github.event.pull_request.number }} \ + --body "The equivalence tests will be updated. Please verify the changes [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})." diff --git a/.github/workflows/manual-equivalence-tests.yml b/.github/workflows/equivalence-test-manual-update.yml similarity index 55% rename from .github/workflows/manual-equivalence-tests.yml rename to .github/workflows/equivalence-test-manual-update.yml index 1fa5dcf9a99c..615c43a10d17 100644 --- a/.github/workflows/manual-equivalence-tests.yml +++ b/.github/workflows/equivalence-test-manual-update.yml @@ -1,4 +1,4 @@ -name: manual-equivalence-tests +name: equivalence-tests-manual on: workflow_dispatch: @@ -7,9 +7,9 @@ on: type: string description: "Which branch should be updated?" required: true - terraform-version: + new-branch: type: string - description: "Terraform version to run against (no v prefix, eg. 1.4.4)." + description: "Name of new branch to be created for the review." required: true equivalence-test-version: type: string @@ -18,7 +18,8 @@ on: required: true permissions: - contents: write # We push updates to the equivalence tests back into the repository. + contents: write + pull-requests: write jobs: run-equivalence-tests: @@ -28,10 +29,23 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.target-branch }} + + - name: Determine Go version + id: go + uses: ./.github/actions/go-version + + - name: Install Go toolchain + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + with: + go-version: ${{ steps.go.outputs.version }} + cache-dependency-path: go.sum + - uses: ./.github/actions/equivalence-test with: - target-terraform-version: ${{ inputs.terraform-version }} - target-terraform-branch: ${{ inputs.target-branch }} target-equivalence-test-version: ${{ inputs.equivalence-test-version }} target-os: linux target-arch: amd64 + current-branch: ${{ inputs.target-branch }} + new-branch: ${{ inputs.new-branch }} + reviewers: ${{ github.actor }} + message: "Update equivalence test golden files." diff --git a/.github/workflows/equivalence-test-update.yml b/.github/workflows/equivalence-test-update.yml new file mode 100644 index 000000000000..988fbf94d9b2 --- /dev/null +++ b/.github/workflows/equivalence-test-update.yml @@ -0,0 +1,66 @@ +name: equivalence-test-update + +on: + pull_request_target: + types: [ closed ] + +permissions: + contents: write + pull-requests: write + +jobs: + check: + name: "Should run equivalence tests?" + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.target_branch.outputs.should_run }} + steps: + - name: target_branch + id: target_branch + run: | + merged = ${{ github.event.pull_request.merged }} + target_branch = ${{ github.event.pull_request.base.ref }} + + targets_release_branch = false + if [ "$target_branch" == "main" ]; then + targets_release_branch = true + elif [ "$target_branch" =~ ^v[0-9]+\.[0-9]+$ ]; then + targets_release_branch = true + fi + + should_run=false + if [ "$merged" == "true" ] && [ "$targets_release_branch" == "true" ]; then + should_run=true + fi + + echo "should_run=$should_run" >> ${GITHUB_OUTPUT} + run-equivalence-tests: + name: "Run equivalence tests" + needs: + - check + if: needs.check.outputs.should_run == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ inputs.target-branch }} + + - name: Determine Go version + id: go + uses: ./.github/actions/go-version + + - name: Install Go toolchain + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + with: + go-version: ${{ steps.go.outputs.version }} + cache-dependency-path: go.sum + + - uses: ./.github/actions/equivalence-test + with: + target-equivalence-test-version: ${{ inputs.equivalence-test-version }} + target-os: linux + target-arch: amd64 + current-branch: ${{ github.event.pull_request.base.ref }} + new-branch: equivalence-testing/${{ github.event.pull_request.head.ref }} + reviewers: ${{ github.event.pull_request.merged_by.login }} + message: "Update equivalence test golden files after ${{ github.event.pull_request.url }}." diff --git a/testing/equivalence-tests/README.md b/testing/equivalence-tests/README.md new file mode 100644 index 000000000000..e6e812f7e7c8 --- /dev/null +++ b/testing/equivalence-tests/README.md @@ -0,0 +1,46 @@ +# Equivalence testing + +This directory contains the test cases for the equivalence testing. The +Terraform equivalence tests are E2E tests that are used to verify that the +output of Terraform commands doesn't change in unexpected ways. The tests are +run by comparing the output of the Terraform commands before and after a change +to the codebase. + +## Running the tests + +The equivalence tests are executed by the Terraform equivalence testing +framework. This is built in [github.com/hashicorp/terraform-equivalence-testing](https://github.com/hashicorp/terraform-equivalence-testing). + +To execute the tests you must download the `terraform-equivalence-testing` +binary and execute either the `diff` or `update` command. The `diff` command +will run the tests and output the differences between the current and previous +run. The `update` command will run the tests and update the reference output +files. + +You can also execute the tests directly using the `equivalence-tests-manual` +GitHub action. This action will run the tests against a given branch and +open a PR with the results. + +## Automated testing + +The equivalence tests are run automatically by the Terraform CI system. The +tests are run when every pull request is opened and when every pull request +is closed. + +When pull requests are opened, the tests run the diff command and will comment +on the PR with the results. PR authors should validate any changes to the output +files and make sure that the changes are expected. + +When pull requests are closed, the tests run the update command and open a new +PR with the updated reference output files. PR authors should review the changes +and make sure that the changes are expected before merging the automated PR. + +If the framework detects no changes, the process should be invisible to the PR +author. No comments will be made on the PR and no new PRs will be opened. + +## Writing new tests + +New tests should be written into the `tests` directory. Each test should be +written in a separate directory and should follow the guidelines in the +equivalence testing framework documentation. Any tests added to this directory +will be picked up the CI system and run automatically.