Execute equivalence tests on every pull request (#35976)

.github/actions/equivalence-test/action.yml

@@ -2,14 +2,8 @@
 # SPDX-License-Identifier: BUSL-1.1
 
 name: equivalence-test
-description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests"
+description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests and update the golden files."
 inputs:
-  target-terraform-version:
-    description: "The version of Terraform to use in execution."
-    required: true
-  target-terraform-branch:
-    description: "The branch within this repository to update and compare."
-    required: true
   target-equivalence-test-version:
     description: "The version of the Terraform equivalence tests to use."
     default: "0.3.0"
@@ -19,9 +13,22 @@ inputs:
   target-arch:
     description: "Current architecture"
     default: "amd64"
+  current-branch:
+    description: "What branch are we currently on?"
+    required: true
+  new-branch:
+    description: "Name of new branch to be created for the review."
+    required: true
+  reviewers:
+    description: "Comma-separated list of GitHub usernames to request review from."
+    required: true
+  message:
+    description: "Message to include in the commit."
+    required: true
 runs:
   using: "composite"
   steps:
+
     - name: "download equivalence test binary"
       shell: bash
       run: |
@@ -30,15 +37,13 @@ runs:
           ./bin/equivalence-tests \
           ${{ inputs.target-os }} \
           ${{ inputs.target-arch }}
-    - name: "download terraform binary"
+
+    - name: Build terraform
       shell: bash
-      run: |
-        ./.github/scripts/equivalence-test.sh download_terraform_binary \
-          ${{ inputs.target-terraform-version }} \
-          ./bin/terraform \
-          ${{ inputs.target-os }} \
-          ${{ inputs.target-arch }}
+      run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform
+
     - name: "run and update equivalence tests"
+      id: execute
       shell: bash
       run: |
         ./bin/equivalence-tests update \
@@ -47,15 +52,24 @@ runs:
           --binary=$(pwd)/bin/terraform
         
         changed=$(git diff --quiet -- testing/equivalence-tests/outputs || echo true)
-        if [[ $changed == "true" ]]; then
-          echo "found changes, and pushing new golden files into branch ${{ inputs.target-terraform-branch }}."
-
-          git config user.email "[email protected]"
-          git config user.name "The Terraform Team"
+        echo "changed=$changed" >> "${GITHUB_OUTPUT}"
 
-          git add ./testing/equivalence-tests/outputs
-          git commit -m "Automated equivalence test golden file update for release ${{ inputs.target-terraform-version }}."
-          git push
-        else
-          echo "found no changes, so not pushing any updates."
-        fi
+    - name: "branch, commit, and push changes"
+      if: steps.execute.outputs.changed == 'true'
+      shell: bash
+      run: |
+        git checkout -b ${{ inputs.new-branch }}
+        git add testing/equivalence-tests/outputs
+        git commit -m "Update equivalence test golden files."
+        git push --set-upstream origin ${{ inputs.new-branch }}
+          
+    - name: "create pull request"
+      if: steps.execute.outputs.changed == 'true'
+      shell: bash
+      run: |
+        gh pr create \
+          --base ${{ inputs.current-branch }} \
+          --head ${{ inputs.new-branch }} \
+          --title "Update equivalence test golden files" \
+          --body "This PR updates the equivalence test golden files." \
+          --reviewer ${{ inputs.reviewers }}

.github/scripts/equivalence-test.sh

@@ -26,11 +26,11 @@ Commands:
 
     ./equivalence-test.sh download_equivalence_test_binary 0.3.0 ./bin/terraform-equivalence-testing linux amd64
 
-  download_terraform_binary <version> <target> <os> <arch>
-    download_terraform_binary downloads the terraform release binary for a given
-    version and places it at the target path.
+  build_terraform_binary <target>
+    download_terraform_binary builds the Terraform binary and places it at the
+    target path.
 
-    ./equivalence-test.sh download_terraform_binary 1.4.3 ./bin/terraform linux amd64
+    ./equivalence-test.sh build_terraform_binary ./bin/terraform
 EOF
 }
 
@@ -65,25 +65,17 @@ function download_equivalence_test_binary {
   rm releases.json
 }
 
-function download_terraform_binary {
-  VERSION="${1:-}"
-  TARGET="${2:-}"
-  OS="${3:-}"
-  ARCH="${4:-}"
+function build_terraform_binary {
+  TARGET="${1:-}"
 
-  if [[ -z "$VERSION" || -z "$TARGET" || -z "$OS" || -z "$ARCH" ]]; then
-    echo "missing at least one of [<version>, <target>, <os>, <arch>] arguments"
+  if [[ -z "$TARGET" ]]; then
+    echo "target argument"
     usage
     exit 1
   fi
 
-  mkdir -p zip
-  curl "https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_${OS}_${ARCH}.zip" > "zip/terraform.zip"
-
-  mkdir -p bin
-  unzip -p "zip/terraform.zip" terraform > "$TARGET"
+  go build -o "$TARGET" ./
   chmod u+x "$TARGET"
-  rm -r zip
 }
 
 function get_target_branch {
@@ -142,14 +134,14 @@ function main {
       download_equivalence_test_binary "$2" "$3" "$4" "$5"
 
       ;;
-    download_terraform_binary)
-      if [ "${#@}" != 5 ]; then
+    build_terraform_binary)
+      if [ "${#@}" != 2 ]; then
         echo "invalid number of arguments"
         usage
         exit 1
       fi
 
-      download_terraform_binary "$2" "$3" "$4" "$5"
+      build_terraform_binary "$2"
 
       ;;
     *)

.github/workflows/equivalence-test-diff.yml

@@ -0,0 +1,70 @@
+name: equivalence-test-diff
+
+on:
+  pull_request:
+    types:
+      - synchronize
+      - ready_for_review
+      - reopened
+      - synchronize
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  equivalence-test-diff:
+    name: "Equivalence Test Diff"
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Fetch source code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Determine Go version
+        id: go
+        uses: ./.github/actions/go-version
+
+      - name: Install Go toolchain
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache-dependency-path: go.sum
+
+      - name: Download testing framework
+        shell: bash
+        run: |
+          ./.github/scripts/equivalence-test.sh download_equivalence_test_binary \
+            0.3.0 \
+            ./bin/equivalence-tests \
+            linux \
+            amd64
+
+      - name: Build terraform
+        shell: bash
+        run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform
+
+      - name: Run equivalence tests
+        id: equivalence-tests
+        shell: bash {0} # we want to capture the exit code
+        run: |
+          ./bin/equivalence-tests diff \
+              --tests=testing/equivalence-tests/tests \
+              --goldens=testing/equivalence-tests/outputs \
+              --binary=$(pwd)/bin/terraform
+          echo "exit-code=$?" >> "${GITHUB_OUTPUT}"
+
+      - name: Equivalence tests failed
+        if: steps.equivalence-tests.outputs.exit-code == 1 # 1 is the exit code for failure
+        shell: bash
+        run: |
+          gh pr comment ${{ github.event.pull_request.number }} \
+            --body "The equivalence tests failed. Please investigate [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})."
+          exit 1 # fail the job
+
+      - name: Equivalence tests changed
+        if: steps.equivalence-tests.outputs.exit-code == 2 # 2 is the exit code for changed
+        shell: bash
+        run: |
+          gh pr comment ${{ github.event.pull_request.number }} \
+            --body "The equivalence tests will be updated. Please verify the changes [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})."

.github/workflows/manual-equivalence-tests.yml → .github/workflows/equivalence-test-manual-update.yml

@@ -1,4 +1,4 @@
-name: manual-equivalence-tests
+name: equivalence-tests-manual
 
 on:
   workflow_dispatch:
@@ -7,9 +7,9 @@ on:
         type: string
         description: "Which branch should be updated?"
         required: true
-      terraform-version:
+      new-branch:
         type: string
-        description: "Terraform version to run against (no v prefix, eg. 1.4.4)."
+        description: "Name of new branch to be created for the review."
         required: true
       equivalence-test-version:
         type: string
@@ -18,7 +18,8 @@ on:
         required: true
 
 permissions:
-  contents: write # We push updates to the equivalence tests back into the repository.
+  contents: write
+  pull-requests: write
 
 jobs:
   run-equivalence-tests:
@@ -28,10 +29,23 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ inputs.target-branch }}
+
+      - name: Determine Go version
+        id: go
+        uses: ./.github/actions/go-version
+
+      - name: Install Go toolchain
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache-dependency-path: go.sum
+
       - uses: ./.github/actions/equivalence-test
         with:
-          target-terraform-version: ${{ inputs.terraform-version }}
-          target-terraform-branch: ${{ inputs.target-branch }}
           target-equivalence-test-version: ${{ inputs.equivalence-test-version }}
           target-os: linux
           target-arch: amd64
+          current-branch: ${{ inputs.target-branch }}
+          new-branch: ${{ inputs.new-branch }}
+          reviewers: ${{ github.actor }}
+          message: "Update equivalence test golden files."

.github/workflows/equivalence-test-update.yml

@@ -0,0 +1,66 @@
+name: equivalence-test-update
+
+on:
+  pull_request_target:
+    types: [ closed ]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  check:
+    name: "Should run equivalence tests?"
+    runs-on: ubuntu-latest
+    outputs:
+        should_run: ${{ steps.target_branch.outputs.should_run }}
+    steps:
+      - name: target_branch
+        id: target_branch
+        run: |
+          merged = ${{ github.event.pull_request.merged }}
+          target_branch = ${{ github.event.pull_request.base.ref }}
+          
+          targets_release_branch = false
+          if [ "$target_branch" == "main" ]; then
+            targets_release_branch = true
+          elif [ "$target_branch" =~ ^v[0-9]+\.[0-9]+$ ]; then
+            targets_release_branch = true
+          fi
+          
+          should_run=false
+          if [ "$merged" == "true" ] && [ "$targets_release_branch" == "true" ]; then
+            should_run=true
+          fi
+          
+          echo "should_run=$should_run" >> ${GITHUB_OUTPUT}
+  run-equivalence-tests:
+    name: "Run equivalence tests"
+    needs:
+      - check
+    if: needs.check.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.target-branch }}
+
+      - name: Determine Go version
+        id: go
+        uses: ./.github/actions/go-version
+
+      - name: Install Go toolchain
+        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        with:
+          go-version: ${{ steps.go.outputs.version }}
+          cache-dependency-path: go.sum
+
+      - uses: ./.github/actions/equivalence-test
+        with:
+          target-equivalence-test-version: ${{ inputs.equivalence-test-version }}
+          target-os: linux
+          target-arch: amd64
+          current-branch: ${{ github.event.pull_request.base.ref }}
+          new-branch: equivalence-testing/${{ github.event.pull_request.head.ref }}
+          reviewers: ${{ github.event.pull_request.merged_by.login }}
+          message: "Update equivalence test golden files after ${{ github.event.pull_request.url }}."