feat: crunchy deployment

.github/workflows/.dbdeployer.yml

@@ -0,0 +1,176 @@
+name: .Crunchy Deploy
+
+on:
+  workflow_call:
+    inputs: ### Required
+      directory:
+        description: Crunchy Chart directory
+        default: 'charts/crunchy'
+        required: false
+        type: string
+      oc_server:
+        default: https://api.silver.devops.gov.bc.ca:6443
+        description: 'OpenShift server'
+        required: false
+        type: string
+      environment:
+        description: Environment name; omit for PRs
+        required: false
+        type: string
+      cluster_environment:
+        description: Cluster environment name, should be dev,test,prod
+        required: false
+        type: string
+        default: 'dev'
+      s3_enabled:
+        description: Enable S3 backups
+        required: false
+        default: true
+        type: boolean
+      values:
+        description: 'Values file'
+        default: 'values.yaml'
+        required: false
+        type: string
+      app_values:
+        description: 'App specific values file which is present inside charts/app'
+        default: 'values.yaml'
+        required: false
+        type: string
+      enabled:
+        description: 'Enable the deployment of the crunchy database, easy switch to turn it on/off'
+        default: true
+        required: false
+        type: boolean
+      timeout-minutes:
+        description: 'Timeout minutes'
+        default: 20
+        required: false
+        type: number
+      triggers:
+        description: Paths used to trigger a database deployment
+        required: false
+        type: string
+      params:
+        description: 'Extra parameters to pass to helm upgrade'
+        default: ''
+        required: false
+        type: string
+    secrets:
+      oc_namespace:
+        description: OpenShift namespace
+        required: true
+      oc_token:
+        description: OpenShift token
+        required: true
+      s3_access_key:
+        description: S3 access key
+        required: false
+      s3_secret_key:
+        description: S3 secret key
+        required: false
+jobs:
+  deploy_db:
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+    runs-on: ubuntu-24.04
+    if: ${{ inputs.enabled }}
+    name: Deploy Or Upgrade Crunchy DB
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install CLI tools from OpenShift Mirror
+        uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4.14.37"
+      - uses: bcgov-nr/[email protected]
+        id: triggers
+        with:
+          triggers: ${{ inputs.triggers }}
+      - name: Validate Inputs
+        if: steps.triggers.outputs.triggered == 'true'
+        shell: bash
+        run: |
+          if [ ${{ inputs.s3_enabled }} == true ]; then
+            echo "S3 ie enabled for backups, checking for mandatory secrets"
+            if [ ! "${{ secrets.s3_access_key }}" ]; then
+              echo "S3 access key not found"
+              exit 1
+            fi
+            if [ ! "${{ secrets.s3_secret_key }}" ]; then
+              echo "S3 secret key not found"
+              exit 1
+            fi
+          fi
+
+      - name: OC Login
+        shell: bash
+        run: |
+          # OC Login
+          OC_TEMP_TOKEN=$(curl -k -X POST ${{ inputs.oc_server }}/api/v1/namespaces/${{ secrets.oc_namespace }}/serviceaccounts/pipeline/token --header "Authorization: Bearer ${{ secrets.oc_token }}" -d '{"spec": {"expirationSeconds": 600}}' -H 'Content-Type: application/json; charset=utf-8' | jq -r '.status.token' )
+
+          oc login --token=$OC_TEMP_TOKEN --server=${{ inputs.oc_server }}
+          oc project ${{ secrets.oc_namespace }} # Safeguard!
+
+      - name: Deploy Database
+        if: steps.triggers.outputs.triggered == 'true'
+        working-directory: ${{ inputs.directory }}
+        shell: bash
+        run: |
+          echo 'Deploying crunchy helm chart'
+          if [ ${{ inputs.s3_enabled }} == true ]; then
+            helm upgrade ${{ inputs.params }} --install --wait --set crunchy.pgBackRest.s3.enabled=true \
+             --set-string crunchy.pgBackRest.s3.accessKey=${{ secrets.s3_access_key }} \
+              --set-string crunchy.pgBackRest.s3.secretKey=${{ secrets.s3_secret_key }} \
+              --values ${{ inputs.values }} postgres  .
+          else
+            helm upgrade ${{ inputs.params }} --install --wait --values ${{ inputs.values }} postgres .
+          fi
+
+      - name: Add PR specific user to Crunchy DB # only for PRs
+        shell: bash
+        if: github.event_name == 'pull_request'
+        run: |
+            echo 'Adding PR specific user to Crunchy DB'
+            NEW_USER='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
+            CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy-${{ inputs.cluster_environment }} -o json | jq '.spec.users')
+            echo "${CURRENT_USERS}"
+
+            # check if current_users already contains the new_user
+            if echo "${CURRENT_USERS}" | jq -e ".[] | select(.name == \"app-${{ github.event.number }}\")" > /dev/null; then
+              echo "User already exists"
+              exit 0
+            fi
+
+            UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson NEW_USER "$NEW_USER" '. + [$NEW_USER]')
+            echo "$UPDATED_USERS"
+            PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
+            echo "$PATCH_JSON"
+            oc patch PostgresCluster/postgres-crunchy-${{ inputs.cluster_environment }} --type=merge -p "${PATCH_JSON}"
+
+            # wait for sometime as it takes time to create the user, query the secret and check if it is created, otherwise wait in a loop for 10 rounds
+            for i in {1..10}; do
+              if oc get secret postgres-crunchy-${{ inputs.cluster_environment }}-pguser-app-${{ github.event.number }} -o jsonpath='{.metadata.name}' > /dev/null; then
+                echo "Secret created"
+                break
+              else
+                echo "Secret not created, waiting for 60 seconds"
+                sleep 60
+              fi
+            done
+
+            # Add public schema and grant to PR user
+            # get primary crunchy pod and remove the role and db
+            CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
+            echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "CREATE SCHEMA IF NOT EXISTS public;"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA public TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON TABLES TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON SEQUENCES TO \"app-${{ github.event.number }}\";"
+            oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -d "app-${{ github.event.number }}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL PRIVILEGES ON FUNCTIONS TO \"app-${{ github.event.number }}\";"
+            # TODO: remove these
+
+
+

.github/workflows/merge-release.yml

@@ -36,10 +36,22 @@ jobs:
       - name: Set PR Output
         run: echo "pr=${{ steps.pr.outputs.pr }}" >> $GITHUB_OUTPUT
 
+  # https://github.com/bcgov/quickstart-openshift
+  crunchy:
+    name: Deploy Crunchy
+    needs: [vars]
+    uses: ./.github/workflows/.dbdeployer.yml
+    with:
+      params: --set global.environment=test
+      cluster_environment: test
+    secrets:
+      oc_namespace: ${{ secrets.OC_NAMESPACE }}
+      oc_token: ${{ secrets.OC_TOKEN }}
+
   # https://github.com/bcgov/quickstart-openshift-helpers
   deploy-test:
     name: Deploy (test)
-    needs: vars
+    needs: [vars, crunchy]
     uses: bcgov/quickstart-openshift-helpers/.github/workflows/[email protected]
     secrets:
       oc_namespace: ${{ secrets.OC_NAMESPACE }}
@@ -58,6 +70,8 @@ jobs:
         --set nats.config.cluster.enabled=true
         --set backup.enabled=true
         --set backup.persistence.size=256Mi
+        # --set useCrunchy=true
+        # --set crunchyClusterEnvironment=test
 
   healthcheck:
     name: Healthcheck Test Deployment

.github/workflows/pr-close.yml

@@ -42,3 +42,50 @@ jobs:
           OC_SERVER: ${{ vars.OC_SERVER }}
           OC_TOKEN: ${{ secrets.OC_TOKEN }}
           PR_NUMBER: ${{ github.event.number }}
+
+  cleanup-crunchy:
+    name: Cleanup Ephemeral Crunchy Data
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      # OC setup
+      - uses: redhat-actions/openshift-tools-installer@v1
+        with:
+          oc: "4"
+
+      # OC Login
+      - run: |
+          # OC Login
+          oc login --token=${{ secrets.OC_TOKEN }} --server=${{ vars.OC_SERVER }}
+          oc project ${{ secrets.OC_NAMESPACE }} # Safeguard!
+      - run: |
+          # check if postgres-crunchy exists or else exit
+          oc get PostgresCluster/postgres-crunchy || exit 0
+          # Remove the user from the crunchy cluster yaml and apply the changes
+          USER_TO_REMOVE='{"databases":["app-${{ github.event.number }}"],"name":"app-${{ github.event.number }}"}'
+
+          echo 'getting current users from crunchy'
+          CURRENT_USERS=$(oc get PostgresCluster/postgres-crunchy -o json | jq '.spec.users')
+          echo "${CURRENT_USERS}"
+
+          # Remove the user from the list, 
+          UPDATED_USERS=$(echo "$CURRENT_USERS" | jq --argjson user "$USER_TO_REMOVE" 'map(select(. != $user))')
+
+          PATCH_JSON=$(jq -n --argjson users "$UPDATED_USERS" '{"spec": {"users": $users}}')
+          oc patch PostgresCluster/postgres-crunchy --type=merge -p "$PATCH_JSON"
+
+          # get primary crunchy pod and remove the role and db
+          CRUNCHY_PG_PRIMARY_POD_NAME=$(oc get pods -l postgres-operator.crunchydata.com/role=master -o json | jq -r '.items[0].metadata.name')
+
+          echo "${CRUNCHY_PG_PRIMARY_POD_NAME}"
+          # Terminate all connections to the database before trying terminate
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'app-${{ github.event.number }}' AND pid <> pg_backend_pid();"
+
+          # Drop the database and role
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "DROP DATABASE \"app-${{ github.event.number }}\" --cascade" 
+
+          oc exec "${CRUNCHY_PG_PRIMARY_POD_NAME}" -- psql -c "DROP ROLE \"app-${{ github.event.number }}\" --cascade" 
+
+          echo "Database and Role for PR is cleaned."
+
+          exit 0

.github/workflows/pr-open.yml

@@ -32,21 +32,34 @@ jobs:
           tag_fallback: latest
           triggers: ('${{ matrix.package }}/')
 
+  # https://github.com/bcgov/quickstart-openshift
+  crunchy:
+    name: Deploy Crunchy
+    needs: [builds]
+    uses: ./.github/workflows/.dbdeployer.yml
+    with:
+      s3_enabled: false # no backups in dev
+      params: --set global.environment=dev
+      cluster_environment: dev
+    secrets:
+      oc_namespace: ${{ secrets.OC_NAMESPACE }}
+      oc_token: ${{ secrets.OC_TOKEN }}
+
   # https://github.com/bcgov/quickstart-openshift-helpers
   deploys:
     name: Deploys
-    needs: [builds]
+    needs: [builds, crunchy]
     uses: bcgov/quickstart-openshift-helpers/.github/workflows/[email protected]
     secrets:
       oc_namespace: ${{ secrets.OC_NAMESPACE }}
       oc_token: ${{ secrets.OC_TOKEN }}
     with:
-      triggers: ('backend/' 'frontend/' 'webeoc/' 'migrations/')
+      params: --set global.useCrunchy=true --set global.crunchyClusterEnvironment=dev
 
   healthcheck:
     name: Healthcheck Deployment
     runs-on: ubuntu-22.04
-    needs: [builds]
+    needs: [builds, crunchy]
     environment:
     timeout-minutes: 15
     if: ${{ ! github.event.pull_request.draft }}

.github/workflows/release-main.yml

@@ -64,10 +64,22 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  # https://github.com/bcgov/quickstart-openshift
+  crunchy:
+    name: Deploy Crunchy
+    needs: [vars]
+    uses: ./.github/workflows/.dbdeployer.yml
+    with:
+      params: --set global.environment=prod
+      cluster_environment: prod
+    secrets:
+      oc_namespace: ${{ secrets.OC_NAMESPACE }}
+      oc_token: ${{ secrets.OC_TOKEN }}
+
   # https://github.com/bcgov/quickstart-openshift-helpers
   deploy-prod:
     name: Deploy (prod)
-    needs: [vars]
+    needs: [vars, crunchy]
     uses: bcgov/quickstart-openshift-helpers/.github/workflows/[email protected]
     secrets:
       oc_namespace: ${{ secrets.OC_NAMESPACE }}
@@ -86,6 +98,8 @@ jobs:
         --set nats.config.cluster.enabled=true
         --set backup.enabled=true
         --set backup.persistence.size=256Mi
+        # --set useCrunchy=true
+        # --set crunchyClusterEnvironment=prod
 
   healthcheck:
     name: Healthcheck Prod Deployment

backend/Dockerfile

@@ -26,5 +26,5 @@ HEALTHCHECK --interval=30s --timeout=3s CMD curl -f http://localhost/:3000
 
 # Non-privileged user
 USER app
-# max old space the heap size, 120MB with 200MB limit in deployment.
-CMD ["--max-old-space-size=120", "/app/dist/main"]
+# max old space the heap size, 500MB 
+CMD ["--max-old-space-size=500", "/app/dist/main"]