Sam/15 16 nix package, build and test of 15.8 and 16.3 (#1266)

* feat: build and flake check of pg 16.3 with exts/wrappers * pg_partman test 15/16 compat * merge sql interface test * tests: build test and cache both versions * chore: run checks individually * pg 15 and 16 packer/ansible/ghactions (#1268) * fix: reformat ec2 cleanup commands (#1267) Co-authored-by: Sam Rose <[email protected]> * feat: build and flake check of pg 16.3 with exts/wrappers * pg_partman test 15/16 compat * merge sql interface test * tests: build test and cache both versions * chore: run checks individually * feat: realease 15 and 16 to staging * chore: update versions * chore: make yq available * chore: run yq from nix * chore: more setup for staging AMI * fix: yq usage * chore: shell vars * fix: When --init none is used, only users who can elevate to sudo privileges can run Nix * fix: no -i * fix: quote correction * fix: newline extra quotes * fix: no need for pg major version on packer * fix: postgresql_major * fix: ql * fix: no ansible args in stage to invocation * fix: unique val * fix: adjustments to build scripts * chore: env var handling * fix: bump to build * chore: set up more required vars * chore: bump var * feat: pg 16 debug symbols * feat: matrix pg versions build on testinfra * feat: matrix on Test Database * chore: running nix in the right context * feat: just use existing Dockerfile + pg version * chore: refer to var * fix: read name without including quotes * chore: try format function * fix: strip quotes from version number * chore: env var * fix: pg client * fix * fix: try to use psql from our own corresponding pkg * fix: try psql from ppa * fix: dbmate per pg version * build dbmate and then install client * fix: account for architecture * chore: limit changes detection migrations/schema.sql * missing docker compose call * ore: drop tests while investigating * test: try on pg15 only * chore: schema needs update * chore: now run on all versions in matrix * test: trying a version of schema per major pg version as there are type diffs --------- Co-authored-by: Sam Rose <[email protected]> Co-authored-by: Oliver Rice <[email protected]> * chore: rm dead code * chore: resolve versions from ansible/vars.yml instead of packer file create a matrix build for 15 16 (or how ever many versions there are) * fix: don not lose if statements used for hotfix when needed * chore: cleanup --------- Co-authored-by: Sam Rose <[email protected]> Co-authored-by: Oliver Rice <[email protected]>
supabase · Oct 17, 2024 · cfbb171 · cfbb171
1 parent 578e275
commit cfbb171
Show file tree

Hide file tree

Showing 39 changed files with 2,957 additions and 148 deletions.
diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml
@@ -8,12 +8,31 @@ on:
     paths:
       - '.github/workflows/ami-release-nix.yml'
       - 'common-nix.vars.pkr.hcl'
+      - 'ansible/vars.yml'
   workflow_dispatch:
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Set PostgreSQL versions
+        id: set-versions
+        run: |
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
+
   build:
+    needs: prepare
     strategy:
       matrix:
+        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
         include:
           - runner: arm-runner
             arch: arm64
@@ -31,42 +50,55 @@ jobs:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
+      - uses: DeterminateSystems/nix-installer-action@main
+
       - name: Run checks if triggered manually
         if: ${{ github.event_name == 'workflow_dispatch' }}
-        # Update `ci.yaml` too if changing constraints.
         run: |
-          SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl)
+          SUFFIX=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/')
           if [[ -z $SUFFIX ]] ; then
             echo "Version must include non-numeric characters if built manually."
             exit 1
           fi
 
-      # extensions are build in nix prior to this step
-      # so we can just use the binaries from the nix store
-      # for postgres, extensions and wrappers
+      - name: Set PostgreSQL version environment variable
+        run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
+
+      - name: Generate common-nix.vars.pkr.hcl
+        run: |
+          PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          PG_VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
+          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
+          # Ensure there's a newline at the end of the file
+          echo "" >> common-nix.vars.pkr.hcl
 
       - name: Build AMI stage 1
+        env:
+          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
         run: |
           packer init amazon-arm64-nix.pkr.hcl
           GIT_SHA=${{github.sha}}
-          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments="  amazon-arm64-nix.pkr.hcl
+          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
 
       - name: Build AMI stage 2
+        env:
+          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
         run: |
           packer init stage2-nix-psql.pkr.hcl
           GIT_SHA=${{github.sha}}
-          packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
+          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl)
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
 
       - name: Create nix flake revision tarball
         run: |
           GIT_SHA=${{github.sha}}
-          MAJOR_VERSION=$(echo "${{ steps.process_release_version.outputs.version }}" | cut -d. -f1)
+          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
           echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -84,17 +116,13 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
+            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
             manifest-playbook.yml
 
       - name: Upload nix flake revision to s3 staging
         run: |
           aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
 
-      #Our self hosted github runner already has permissions to publish images 
-      #but they're limited to only that; 
-      #so if we want s3 access we'll need to config credentials with the below steps 
-      # (which overwrites existing perms) after the ami build
-
       - name: configure aws credentials - prod
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -107,6 +135,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
+            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
             manifest-playbook.yml
     
       - name: Upload nix flake revision to s3 prod
@@ -130,12 +159,12 @@ jobs:
           SLACK_MESSAGE: 'Building Postgres AMI failed'
           SLACK_FOOTER: ''
 
-      - name: Cleanup resources on build cancellation
+      - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
-          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {}
+          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
-          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {}
+          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml
@@ -11,8 +11,28 @@ permissions:
   id-token: write
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Set PostgreSQL versions
+        id: set-versions
+        run: |
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
+
   publish-staging:
+    needs: prepare
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
 
     steps:
       - name: Checkout Repo
@@ -21,7 +41,8 @@ jobs:
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
             VERSION=${{ inputs.postgresVersion }}
           fi
@@ -58,6 +79,10 @@ jobs:
   publish-prod:
     runs-on: ubuntu-latest
     if: github.ref_name == 'develop' || contains( github.ref, 'release' )
+    needs: prepare
+    strategy:
+      matrix:
+        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
 
     steps:
       - name: Checkout Repo
@@ -66,10 +91,8 @@ jobs:
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
-            VERSION=${{ inputs.postgresVersion }}
-          fi
+          VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           echo "version=$VERSION" >> "$GITHUB_OUTPUT"
           echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT"
 

diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml
@@ -7,7 +7,6 @@ on:
       - release/*
     paths:
       - '.github/workflows/publish-nix-pgupgrade-scripts.yml'
-      - 'common-nix.vars.pkr.hcl'
   workflow_dispatch:
     inputs:
       postgresVersion:
@@ -18,17 +17,40 @@ permissions:
   id-token: write
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Set PostgreSQL versions
+        id: set-versions
+        run: |
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
+
   publish-staging:
+    needs: prepare
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
 
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
+      - uses: DeterminateSystems/nix-installer-action@main
+
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
+          VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
             VERSION=${{ inputs.postgresVersion }}
           fi
@@ -61,20 +83,24 @@ jobs:
           SLACK_FOOTER: ''
 
   publish-prod:
+    needs: prepare
     runs-on: ubuntu-latest
     if: github.ref_name == 'develop' || contains( github.ref, 'release' )
 
+    strategy:
+      matrix:
+        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
+
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          if [[ "${{ inputs.postgresVersion }}" != "" ]]; then
-            VERSION=${{ inputs.postgresVersion }}
-          fi
+          VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
           echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create a tarball containing pg_upgrade scripts

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -8,9 +8,27 @@ on:
   workflow_dispatch:
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Set PostgreSQL versions
+        id: set-versions
+        run: |
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
+
   build:
+    needs: prepare
     strategy:
       matrix:
+        postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }}
         include:
           - runner: [self-hosted, X64]
             arch: amd64
@@ -23,14 +41,36 @@ jobs:
       POSTGRES_PASSWORD: password
     steps:
       - uses: actions/checkout@v3
+
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Set PostgreSQL version environment variable
+        run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
+
+      - name: Strip quotes from pg major and set env var
+        run: |
+          stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/')
+          echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV
+
+      - name: Generate common-nix.vars.pkr.hcl
+        run: |
+          PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
+          PG_VERSION=$(echo $PG_VERSION | tr -d '"')  # Remove any surrounding quotes
+          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
+          # Ensure there's a newline at the end of the file
+          echo "" >> common-nix.vars.pkr.hcl
+
       - id: settings
         # Remove spaces and quotes to get the raw version string
         run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT
 
-      - id: args
-        uses: mikefarah/yq@master
-        with:
-          cmd: yq 'to_entries | map(select(.value|type == "!!str")) |  map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
+      - name: Generate args
+        id: args
+        run: |
+          ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml)
+          echo "result<<EOF" >> $GITHUB_OUTPUT
+          echo "$ARGS" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
 
       - run: docker context create builders
       - uses: docker/setup-buildx-action@v3
@@ -40,7 +80,7 @@ jobs:
         with:
           load: true
           context: .
-          file: "Dockerfile-156"
+          file: Dockerfile-${{ env.PGMAJOR }}
           target: production
           build-args: |
             ${{ steps.args.outputs.result }}
@@ -57,10 +97,13 @@ jobs:
           -p ${{ env.POSTGRES_PORT }}:5432 \
           --name supabase_postgres \
           -d supabase/postgres:${{ steps.settings.outputs.postgres-version }}
+
       - name: Install psql
         run: |
+          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
           sudo apt update
-          sudo apt install -y --no-install-recommends postgresql-client
+          sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }}
 
       - name: Install pg_prove
         run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP
@@ -107,11 +150,15 @@ jobs:
           PGUSER: supabase_admin
           PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
 
+      - name: Update Dockerfile.dbmate version
+        run: |
+          sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate
+
       - name: verify schema.sql is committed
         run: |
           docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit
-          if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then
-            echo "Detected uncommitted changes after build. See status below:"
-            git diff
+          if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then
+            echo "Detected changes in schema.sql:"
+            git diff migrations/schema-${{ env.PGMAJOR }}.sql
             exit 1
           fi