Terraform + Ansible Github Actions (#1)

Specifically we have added support for the following:
* CD of Terraform IaC deployment via Github Actions
* CD of ansible configuration provisioning via GH Actions
* Update pending PR with comments stating the outcome of the terraform
and ansible deployment tasks
* Dynamic ansible inventory file generation using terraform as source
* Provisioning of secrets and host keys dynamically
* Support for deploying clickhouse to AWS via terraform and configured
via ansible
* Support for doing CD of oonidataapi with AWS ECS using terraform

---------

Co-authored-by: OONI Github Actions Bot <[email protected]>
Co-authored-by: Simone Basso <[email protected]>

.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.github/workflows/check_deploy.yml

@@ -0,0 +1,219 @@
+# For docs on this see:
+# * https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+# * https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+# * https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=synchronize#pull_request
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize # when commits are pushed to the PR
+      - reopened
+      - edited # title or body of a pull request was edited, or the base branch of a pull request was changed
+
+env:
+  tf_actions_working_dir: "./tf/environments/production"
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  ansible:
+    needs: terraform
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.tf_actions_working_dir }}/ansible
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Ansible
+        run: |
+          sudo apt update
+          sudo apt install software-properties-common
+          sudo add-apt-repository --yes --update ppa:ansible/ansible-9
+          sudo apt install -y ansible
+
+      - name: Write devops ssh key to .ssh
+        run: |
+          mkdir -p ~/.ssh/
+          chmod 700 ~/.ssh/
+          echo "${{ secrets.AWS_SSH_KEY }}" > ~/.ssh/ooni-devops-prod.pem
+          chmod 600 ~/.ssh/ooni-devops-prod.pem
+
+      - name: Run Ansible Playbook
+        id: playbook
+        env:
+          ANSIBLE_SSH_ARGS: "-o UserKnownHostsFile=known_hosts"
+        run: |
+          echo "ansible_playbook<<EOF" >> "$GITHUB_OUTPUT"
+          echo "\$ ansible-playbook playbook.yml --check --diff -i inventory.ini" >> "$GITHUB_OUTPUT"
+          ansible-playbook playbook.yml --check --diff -i inventory.ini --key-file ~/.ssh/ooni-devops-prod.pem 2>&1 | tee -a "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+        continue-on-error: true
+
+      # This can be uncommmented to make it possible to ssh into the container to debug the run
+      #- name: Setup tmate session
+      #  uses: mxschmitt/action-tmate@v3
+
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const commentTitle = "Ansible Run Output";
+            const ansiblePlaybookOutput = `${{ steps.playbook.outputs.ansible_playbook}}`;
+            const parts = ansiblePlaybookOutput.split(/PLAY RECAP \*+/);
+            const ansiblePlaybookRecap = parts.length > 1 ? parts[1].trim() : '';
+
+            const commentBody = `
+            #### Ansible Playbook Recap 🔍
+
+            \`\`\`\n
+            ${ansiblePlaybookRecap}
+            \`\`\`
+
+            #### Ansible playbook output 📖\`${{ steps.playbook.outcome }}\`
+
+            <details><summary>Show Execution</summary>
+
+            \`\`\`\n
+            ${ansiblePlaybookOutput}
+            \`\`\`
+
+            </details>
+
+            |                   |                                    |
+            |-------------------|------------------------------------|
+            | Pusher            | @${{ github.actor }}               |
+            | Action            |  ${{ github.event_name }}          |
+            | Working Directory |  ${{ env.tf_actions_working_dir }} |
+            | Workflow          |  ${{ github.workflow }}            |
+            | Last updated      |  ${(new Date()).toUTCString()}     |
+            `;
+
+            // Call the script to write the comment
+            const script = require('./scripts/ghactions/comment-on-pr.js');
+            await script({github, context, core, commentTitle, commentBody});
+
+  terraform:
+    runs-on: ubuntu-latest
+    if: ${{ !startsWith(github.event.head_commit.message, 'skip-terraform:') }}
+    defaults:
+      run:
+        working-directory: ${{ env.tf_actions_working_dir }}
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      TF_VAR_aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      TF_VAR_aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Terraform
+        run: |
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+              | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install terraform
+
+      - name: Terraform fmt
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Init
+        id: init
+        run: terraform init
+
+      - name: Terraform Validate
+        id: validate
+        run: |
+          echo "terraform_validate<<EOF" >> "$GITHUB_OUTPUT"
+          echo "\$ terraform validate" >> "$GITHUB_OUTPUT"
+          terraform validate -no-color | tee -a "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+
+      - name: Terraform Plan
+        id: plan
+        run: |
+          echo "terraform_plan<<EOF" >> "$GITHUB_OUTPUT"
+          echo "\$ terraform plan" >> "$GITHUB_OUTPUT"
+          terraform plan -no-color | tee -a "$GITHUB_OUTPUT"
+          echo "EOF" >> "$GITHUB_OUTPUT"
+        continue-on-error: true
+
+      # Temporarily disabled, probably should be moved to a deploy action with stricter checks
+      #- name: Terraform Apply
+      #  id: apply
+      #  run: |
+      #    echo "terraform_apply<<EOF" >> "$GITHUB_OUTPUT"
+      #    echo "\$ terraform apply -auto-approve" >> "$GITHUB_OUTPUT"
+      #    terraform apply -auto-approve -no-color | tee -a "$GITHUB_OUTPUT"
+      #    echo "EOF" >> "$GITHUB_OUTPUT"
+      #  continue-on-error: true
+
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const terraformPlanOutput = `${{ steps.plan.outputs.terraform_plan }}`;
+            const terraformApplyOutput = `${{ steps.apply.outputs.terraform_apply }}`;
+
+            const terraformPlanPlanLine = terraformPlanOutput.split('\n').find(line => line.startsWith('Plan:'));
+            const terraformApplyPlanLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Plan:'));
+            const terraformApplyApplyLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Apply complete!'));
+
+            const commentTitle = "Terraform Run Output";
+            const commentBody = `
+            #### Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+            #### Initialization ⚙️\`${{ steps.init.outcome }}\`
+            #### Validation 🤖\`${{ steps.validate.outcome }}\`
+            <details><summary>Validation Output</summary>
+
+            \`\`\`\n
+            ${{ steps.validate.outputs.terraform_validate }}
+            \`\`\`
+
+            </details>
+
+            #### Plan 📖\`${{ steps.plan.outcome }}\`
+            * **${terraformPlanPlanLine}**
+
+            <details><summary>Show Plan</summary>
+
+            \`\`\`\n
+            ${terraformPlanOutput}
+            \`\`\`
+
+            </details>
+
+            #### Apply 📖\`${{ steps.apply.outcome }}\`
+            * **${terraformApplyPlanLine}**
+            * **${terraformApplyApplyLine}**
+
+            <details><summary>Show Apply</summary>
+
+            \`\`\`\n
+            ${terraformApplyOutput}
+            \`\`\`
+
+            </details>
+
+            |                   |                                    |
+            |-------------------|------------------------------------|
+            | Pusher            | @${{ github.actor }}               |
+            | Action            |  ${{ github.event_name }}          |
+            | Working Directory |  ${{ env.tf_actions_working_dir }} |
+            | Workflow          |  ${{ github.workflow }}            |
+            | Last updated      |  ${ (new Date()).toUTCString() }   |
+            `;
+
+            // Call the script to write the comment
+            const script = require('./scripts/ghactions/comment-on-pr.js');
+            await script({github, context, core, commentTitle, commentBody});

.gitignore

@@ -1 +1,16 @@
-.terraform
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Ignore override files as they are usually used to override resources locally
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json

scripts/dump-tables-ch.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -euxo pipefail
+# This is to be run manually on the clickhouse host to dump schemas and table dumps
+# You may want to make some tweaks to the dumping rules in order avoid dumping
+# too much data (eg. fastpath)
+# You should then scp the data over to the target host manually, by running:
+# $ scp * clickhouse-instance2:/var/lib/clickhouse/ooni-dumps/
+TABLES=(
+"fastpath"
+"jsonl"
+"url_priorities"
+"citizenlab"
+"citizenlab_flip"
+"test_groups"
+"accounts"
+"session_expunge"
+"msmt_feedback"
+"fingerprints_dns"
+"fingerprints_http"
+"asnmeta"
+"counters_test_list"
+"counters_asn_test_list"
+"incidents"
+"oonirun"
+)
+
+dump_dir="./dumps"
+current_date=$(date +%Y%m%d)
+
+# Directory to store the dumps
+mkdir -p "$dump_dir"
+
+# Iterate over each table
+for table in "${TABLES[@]}"; do
+    # Define file names for schema and data dump
+    schema_file="${dump_dir}/${current_date}-${table}_schema.sql"
+    data_file="${dump_dir}/${current_date}-${table}_dump.clickhouse"
+
+    # Dump the table schema
+    echo "[+] dumping schema $schema_file"
+    clickhouse-client --query="SHOW CREATE TABLE ${table} FORMAT TabSeparatedRaw" > "$schema_file"
+
+    # Dump the table data in ClickHouse native format
+    echo "[+] dumping table data $data_file"
+    clickhouse-client --query="SELECT * FROM ${table} INTO OUTFILE '${data_file}' FORMAT Native"
+done

scripts/ghactions/comment-on-pr.js

@@ -0,0 +1,44 @@
+module.exports = async ({
+  github,
+  context,
+  core,
+  commentTitle,
+  commentBody,
+}) => {
+  const body = `## ${commentTitle} 🤖
+    ${commentBody}
+    `;
+
+  const prNumber = context.payload.pull_request.number;
+
+  if (prNumber) {
+    const { data: comments } = await github.rest.issues.listComments({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: prNumber,
+    });
+    const existingBotComment = comments.find((comment) => {
+      return (
+        comment.user.type === "Bot" &&
+        comment.body.includes(`## ${commentTitle}`)
+      );
+    });
+
+    if (existingBotComment) {
+      await github.rest.issues.updateComment({
+        issue_number: prNumber,
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        comment_id: existingBotComment.id,
+        body,
+      });
+    } else {
+      await github.rest.issues.createComment({
+        issue_number: prNumber,
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        body,
+      });
+    }
+  }
+};