ci: add helm chart and cicd flow

.ebextensions/01_create_indices.config

@@ -1,4 +1,4 @@
 container_commands:
   01create:
     command: rake db:create
-    leader_only: true
+    leader_only: true

.github/pull_request_template.md

@@ -0,0 +1,10 @@
+## Reason
+
+## Changes
+
+## Test Scope
+
+## Checks
+
+- [ ] Unit tests are included, or please explain why it's not applicable.
+- [ ] Keep pull requests small so they can be easily reviewed.

.github/workflows/ci.yml

@@ -0,0 +1,126 @@
+# Reusable workflow to run lint, test, build, push docker image.
+name: Reusable CI Workflow
+
+on:
+  workflow_call:
+    # GitHub Actions doesn't directly support passing secrets between workflows.
+    # Workflows that call reusable workflows in the same organization or
+    # enterprise can use the inherit keyword to implicitly pass the secrets.
+    # Therefore, we should pass the secret to the workflow_call workflow.
+    # Ref: https://github.com/orgs/community/discussions/23107
+    secrets:
+      HC_GITHUB_SSH_KEY:
+        required: true
+      SLACK_WEBHOOK_URL:
+        required: true
+    inputs:
+      should_push_image:
+        description: 'Whether to push image to ECR'
+        required: true
+        type: boolean
+    outputs:
+      app_version:
+        value: ${{ jobs.test.outputs.app_version }}
+
+jobs:
+  lint:
+    runs-on: [self-hosted, general, small]
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1024
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: pre-commit (PR)
+        if: github.event_name == 'pull_request'
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha}}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha}}
+        run: |
+          python -m pip install pre-commit
+          # the bug for virualenv/setuptools/pip
+          # ref: https://github.com/pypa/setuptools/issues/2353
+          SETUPTOOLS_USE_DISTUTILS=stdlib pre-commit run --from-ref $BASE_SHA --to-ref $HEAD_SHA --all-files || (git --no-pager diff && false)
+
+      - name: pre-commit (push)
+        if: github.event_name == 'push'
+        env:
+          BASE_SHA: ${{ github.event.before}}
+          HEAD_SHA: ${{ github.event.after}}
+        run: |
+          python -m pip install pre-commit
+          SETUPTOOLS_USE_DISTUTILS=stdlib pre-commit run --from-ref $BASE_SHA --to-ref $HEAD_SHA --all-files || (git --no-pager diff && false)
+
+  test:
+    runs-on: [self-hosted, general, small]
+    needs:
+      - lint
+    timeout-minutes: 10
+    outputs:
+      app_version: ${{ steps.app-version.outputs.APP_VERSION }}
+    env:
+      PRIVATE_GH_REPO_SSH_KEY_PATH: /tmp/ssh_key
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v4
+
+      - id: app-version
+        name: Set APP_VERSION
+        run: |
+          APP_VERSION=$(make gen-version)
+          echo "APP_VERSION=$APP_VERSION" >> $GITHUB_ENV
+          echo "APP_VERSION=$APP_VERSION" >> $GITHUB_OUTPUT
+
+      - name: prepare ssh key for private github repo
+        run: |
+          echo "${{ secrets.HC_GITHUB_SSH_KEY }}" > $PRIVATE_GH_REPO_SSH_KEY_PATH
+          mkdir -p ~/.ssh && cp $PRIVATE_GH_REPO_SSH_KEY_PATH ~/.ssh/id_rsa
+
+      - name: build docker images
+        run: |
+          make build-docker-image IMAGE_TAG=$APP_VERSION
+
+      - name: run tests
+        run: |
+          make test
+
+      # Scan the Docker images and packages
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: "${{ env.IMAGE_NAME }}:${{ env.APP_VERSION }}"
+          format: 'table'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH'
+        env:
+          IMAGE_NAME: gofreight/csl
+
+      - name: push docker images
+        if: ${{ inputs.should_push_image }}
+        run: |
+          aws ecr describe-repositories --repository-names $IMAGE_NAME > /dev/null 2>&1 || aws ecr create-repository --repository-name $IMAGE_NAME
+          aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com
+          make push-docker-image IMAGE_TAG=$APP_VERSION
+        env:
+          AWS_ACCOUNT_ID: 478041131377
+          AWS_REGION: us-west-2
+          IMAGE_NAME: gofreight/csl
+
+      - name: tear down
+        run: |
+          make tear-down
+
+      - name: slack notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_COLOR: '#ff0000'
+          SLACK_MESSAGE: |
+            Repository: ${{ github.repository }}

.github/workflows/pr-check.yml

@@ -0,0 +1,12 @@
+name: PR Validation
+
+on: [pull_request]
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+    secrets:
+      HC_GITHUB_SSH_KEY: ${{ secrets.HC_GITHUB_SSH_KEY }}  # required for install private package from github private repo with SSH key
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PR_CI_WEBHOOK_URL }}
+    with:
+      should_push_image: false

.github/workflows/release-production.yml

@@ -0,0 +1,36 @@
+name: Release for Production
+
+on:
+  release:
+    types:
+      - created
+  workflow_dispatch:
+
+jobs:
+  deploy-production:
+    runs-on: [self-hosted, general, small]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: release
+          fetch-tags: true
+
+      - name: Validate if GITHUB_RES belongs to Git Tag
+        run: |
+          echo $GITHUB_REF
+          git describe --tags $GITHUB_REF
+
+      - name: Release the specified git tag to production
+        run: |
+          git fetch --unshallow --tags
+          git reset --hard $GITHUB_REF
+          git push -f origin HEAD:release
+
+      - name: slack notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_CI_CRITICAL_WEBHOOK_URL }}
+          SLACK_COLOR: '#ff0000'
+          SLACK_MESSAGE: |
+            Repository: ${{ github.repository }}

.github/workflows/release-staging.yml

@@ -0,0 +1,24 @@
+name: Release for Staging
+
+on:
+  push:
+    branches:
+      - "main"
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+    secrets:
+      HC_GITHUB_SSH_KEY: ${{ secrets.HC_GITHUB_SSH_KEY }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GENERAL_CI_WEBHOOK_URL }}
+    with:
+      should_push_image: true
+
+  deploy-staging:
+    needs: ci
+    uses: ./.github/workflows/update-image-tag.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GENERAL_CI_WEBHOOK_URL }}
+    with:
+      commit_sha: ${{ github.sha }}
+      image_tag: ${{ needs.ci.outputs.app_version }}

.github/workflows/update-image-tag.yml

@@ -0,0 +1,54 @@
+# Reuseable workflow to update image tag in values-version.yaml
+name: Update Image Tag Workflow
+
+on:
+  workflow_call:
+    # GitHub Actions doesn't directly support passing secrets between workflows.
+    # Workflows that call reusable workflows in the same organization or
+    # enterprise can use the inherit keyword to implicitly pass the secrets.
+    # Therefore, we should pass the secret to the workflow_call workflow.
+    # Ref: https://github.com/orgs/community/discussions/23107
+    secrets:
+      SLACK_WEBHOOK_URL:
+        required: true
+    inputs:
+      commit_sha:
+        description: 'base commit_sha to update'
+        required: true
+        type: string
+      image_tag:
+        description: 'Image tag to update'
+        required: true
+        type: string
+
+jobs:
+  update_image_tag:
+    runs-on: [self-hosted, general, small]
+    steps:
+      - name: Checkout commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit_sha }}
+
+      - name: Update image tag in values-version.yaml
+        run: |
+          make update-image-tag IMAGE_TAG=${{ inputs.image_tag }}
+
+      - name: Commit and push changes
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add deployment/kubernetes/charts/csl/values-version.yaml
+          git commit -m "Update image.tag via GitHub Action"
+          git tag ${{ inputs.image_tag }}
+          git push origin ${{ inputs.image_tag }}
+          git push -f origin HEAD:release-stage
+
+      - name: slack notification
+        if: ${{ failure() }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_COLOR: '#ff0000'
+          SLACK_MESSAGE: |
+            Repository: ${{ github.repository }}

.gitignore

@@ -31,4 +31,8 @@ spec/fixtures/.DS_Store
 # Elasticsearch data
 es_data
 
+# Rails
 vendor/
+
+# Python virtualenv
+.venv

.pre-commit-config.yaml

@@ -0,0 +1,79 @@
+# default_stages: [commit, push]
+fail_fast: false
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      # check and format the syntax
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-json
+        # - id: check-yaml
+        exclude: ^deployment/kubernetes/charts
+      - id: check-toml
+      - id: check-xml
+      - id: double-quote-string-fixer
+      # environment checker
+      - id: check-executables-have-shebangs
+      - id: check-shebang-scripts-are-executable
+      # check the git-related commit or object
+      - id: check-merge-conflict
+      - id: check-added-large-files
+      # check sensitive information
+      - id: debug-statements
+      - id: detect-private-key
+
+  - repo: https://github.com/myint/autoflake
+    rev: v2.2.1
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place
+          - --remove-unused-variables
+          - --remove-all-unused-imports
+          - --ignore-init-module-imports # TODO: remove this
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args:
+          - --profile=black
+          - --filter-files
+
+  # - repo: https://github.com/psf/black
+  #   rev: 23.11.0
+  #   hooks:
+  #     - id: black
+  #       language_version: python3.11
+  #       args:
+  #         - --target-version=py311
+  #         - --line-length=120
+  #         - --skip-string-normalization
+
+  # - repo: https://github.com/zricethezav/gitleaks
+  #   rev: v8.18.1
+  #   hooks:
+  #     - id: gitleaks
+  #       # run the detect mode and show all the leak credentials
+  #       entry: gitleaks detect --verbose --redact
+
+  - repo: https://github.com/hadolint/hadolint
+    rev: v2.12.1-beta
+    hooks:
+      - id: hadolint-docker
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+      - id: pyupgrade
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.6
+    hooks:
+      - id: bandit

.rubocop.yml

@@ -6,4 +6,4 @@ AllCops:
   TargetRubyVersion: 2.6
 
 Layout/EmptyLinesAroundAccessModifier:
-  Enabled: false
+  Enabled: false

.ruby-gemset

@@ -1,2 +1 @@
 csl
-

Gemfile

@@ -6,18 +6,17 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby "~> 2.6"
 
 gem "active_elastic_job"
-gem "aws-sdk", "~> 2" # https://github.com/tawan/active-elastic-job/pull/95
+gem 'aws-sdk', '~> 2'
 gem "elasticsearch"
 gem "elasticsearch-model"
-gem "elasticsearch-persistence", "~> 7.0"
-gem "jbuilder", "~> 2.11"
-gem "puma", "~> 5.3"
+gem "elasticsearch-persistence"
+gem "jbuilder"
+gem "puma"
 gem "rails"
 gem "sanitize"
 gem "htmlentities"
 gem "charlock_holmes"
 gem "iso_country_codes"
-gem "public_suffix"
 
 # Reduces boot times through caching; required in config/boot.rb
 gem "bootsnap", ">= 1.4.2", require: false