From aaff0b3d6014629b2b3dead6d5eedd48a72a3f7c Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 11:23:45 -0700 Subject: [PATCH 1/7] chore: gh action --- .github/workflows/build.yml | 54 +++++++++++++++++++++++++++++ estimation/Dockerfile => Dockerfile | 2 +- helm/database/values.yaml | 22 ++++++++++++ helm/deploy_db.sh | 12 +++++++ requirements.txt | 3 +- 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build.yml rename estimation/Dockerfile => Dockerfile (96%) create mode 100644 helm/database/values.yaml create mode 100755 helm/deploy_db.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..af91419 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,54 @@ +name: Build estimation tool + +on: + push: + branches: '*' + +jobs: + docker-build: + runs-on: ubuntu-latest + name: Build docker image + steps: + - uses: actions/checkout@v3 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/bcgov/cas-estimation-tool + tags: | + type=sha,format=long,prefix= + latest + type=ref,event=pr + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-estimation-tool-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-estimation-tool + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + builder: ${{ steps.buildx.outputs.name }} + push: true + file: Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/estimation/Dockerfile b/Dockerfile similarity index 96% rename from estimation/Dockerfile rename to Dockerfile index f2df209..fad2314 100644 --- a/estimation/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use the official Python image from the Docker Hub -FROM python:3.10-slim +FROM python:3.12-slim # Set environment variables ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/helm/database/values.yaml b/helm/database/values.yaml new file mode 100644 index 0000000..210e840 --- /dev/null +++ b/helm/database/values.yaml @@ -0,0 +1,22 @@ +postgresCluster: + storageSize: 512Mi + postgres: + replicaCount: 2 + pgbouncer: + replicaCount: 2 + + # The "users" value(s) is passed to the crunchy postgres operator to create the database. + # See https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/basic-setup/user-management + users: + - name: postgres + options: "SUPERUSER" + - name: estimation + databases: + - estimation + +gcsBackups: + enable: false + +# To configure a KNP allowing external access, for metabase for example +external-access: + enabled: false diff --git a/helm/deploy_db.sh b/helm/deploy_db.sh new file mode 100755 index 0000000..f8e8bc5 --- /dev/null +++ b/helm/deploy_db.sh @@ -0,0 +1,12 @@ + +set -euo pipefail + +echo "Installing database chart: cas-postgres/cas-postgres-cluster..." + +helm repo add cas-postgres https://bcgov.github.io/cas-postgres/ +helm repo update + +helm upgrade --install --atomic --timeout 1800s \ + --namespace "cf52af-test" \ + --values ./database/values.yaml \ + cas-estimation-db cas-postgres/cas-postgres-cluster --version 1.1.1 diff --git a/requirements.txt b/requirements.txt index 19c263e..cce4aac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,8 @@ Django==5.1.1 idna==3.10 Markdown==3.7 oauthlib==3.2.2 -psycopg==3.2.2 +psycopg==3.1.19 +psycopg-binary==3.1.19 psycopg-pool==3.2.3 python-dotenv==1.0.1 requests==2.32.3 From 2ea47a0b30a91f45ba2af840e705b3c034c7fe9a Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 11:26:31 -0700 Subject: [PATCH 2/7] run action on pr --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af91419..1e9b388 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,7 +2,9 @@ name: Build estimation tool on: push: - branches: '*' + branches: [develop, main] + pull_request: + branches: [develop, main] jobs: docker-build: From eba8fc59de049e945ba72bba2d28a2a9ecf5a89c Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 12:53:51 -0700 Subject: [PATCH 3/7] chore: values --- estimation/estimation/settings.py | 13 ++-- helm/cas-estimation-tool/.helmignore | 23 +++++++ helm/cas-estimation-tool/Chart.yaml | 24 +++++++ .../templates/_helpers.tpl | 62 ++++++++++++++++++ .../templates/deployment.yaml | 64 +++++++++++++++++++ helm/cas-estimation-tool/templates/route.yaml | 20 ++++++ .../templates/service.yaml | 13 ++++ helm/cas-estimation-tool/values.yaml | 1 + helm/database/values.yaml | 2 +- 9 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 helm/cas-estimation-tool/.helmignore create mode 100644 helm/cas-estimation-tool/Chart.yaml create mode 100644 helm/cas-estimation-tool/templates/_helpers.tpl create mode 100644 helm/cas-estimation-tool/templates/deployment.yaml create mode 100644 helm/cas-estimation-tool/templates/route.yaml create mode 100644 helm/cas-estimation-tool/templates/service.yaml create mode 100644 helm/cas-estimation-tool/values.yaml diff --git a/estimation/estimation/settings.py b/estimation/estimation/settings.py index 66ff14e..367206d 100644 --- a/estimation/estimation/settings.py +++ b/estimation/estimation/settings.py @@ -81,7 +81,7 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", - "NAME": os.environ.get("PGDATBASE", "estimation"), # database name + "NAME": os.environ.get("PGDATABASE", "estimation"), # database name "USER": os.environ.get("PGUSER", "postgres"), "PASSWORD": os.environ.get("PGPASSWORD"), "HOST": os.environ.get("PGHOST", "localhost"), @@ -123,12 +123,13 @@ USE_TZ = True -#github details - -GITHUB_CLIENT_ID = os.environ.get('GITHUB_CLIENT_ID', '') -GITHUB_CLIENT_SECRET = os.environ.get('GITHUB_CLIENT_SECRET', '') -GITHUB_REDIRECT_URI = os.environ.get('GITHUB_REDIRECT_URI', 'http://localhost:8000/github/callback/') +# github details +GITHUB_CLIENT_ID = os.environ.get("GITHUB_CLIENT_ID", "") +GITHUB_CLIENT_SECRET = os.environ.get("GITHUB_CLIENT_SECRET", "") +GITHUB_REDIRECT_URI = os.environ.get( + "GITHUB_REDIRECT_URI", "http://localhost:8000/github/callback/" +) # Static files (CSS, JavaScript, Images) diff --git a/helm/cas-estimation-tool/.helmignore b/helm/cas-estimation-tool/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/helm/cas-estimation-tool/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/cas-estimation-tool/Chart.yaml b/helm/cas-estimation-tool/Chart.yaml new file mode 100644 index 0000000..35b765d --- /dev/null +++ b/helm/cas-estimation-tool/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: cas-estimation-tool +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/helm/cas-estimation-tool/templates/_helpers.tpl b/helm/cas-estimation-tool/templates/_helpers.tpl new file mode 100644 index 0000000..e470d11 --- /dev/null +++ b/helm/cas-estimation-tool/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cas-estimation-tool.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cas-estimation-tool.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cas-estimation-tool.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cas-estimation-tool.labels" -}} +helm.sh/chart: {{ include "cas-estimation-tool.chart" . }} +{{ include "cas-estimation-tool.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cas-estimation-tool.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cas-estimation-tool.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cas-estimation-tool.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cas-estimation-tool.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/helm/cas-estimation-tool/templates/deployment.yaml b/helm/cas-estimation-tool/templates/deployment.yaml new file mode 100644 index 0000000..089b8a4 --- /dev/null +++ b/helm/cas-estimation-tool/templates/deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "cas-estimation-tool.fullname" . }} + labels: {{ include "cas-estimation-tool.labels" . | nindent 4 }} +spec: + selector: + matchLabels: {{ include "cas-estimation-tool.selectorLabels" . | nindent 6 }} + replicas: 1 + template: + metadata: + labels: {{ include "cas-estimation-tool.labels" . | nindent 8 }} + spec: + imagePullSecrets: + - name: dockerhub-registry + containers: + - name: {{ .Chart.Name }} + image: "ghcr.io/bcgov/cas-estimation-tool:latest" + imagePullPolicy: Always + env: + - name: GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: "cas-estimation-tool-github" + key: client_id + - name: GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "cas-estimation-tool-github" + key: client_secret + - name: TEAM_MEMBERS + valueFrom: + secretKeyRef: + name: "cas-estimation-tool-github" + key: team_members + - name: PGDATABASE + value: estimation + - name: PGUSER + valueFrom: + secretKeyRef: + name: "{{ .Values.databaseSecretName }}" + key: user + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Values.databaseSecretName }}" + key: password + - name: PGHOST + valueFrom: + secretKeyRef: + name: "{{ .Values.databaseSecretName }}" + key: host + - name: PGPORT + value: "5432" + + ports: + - containerPort: 8000 + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 25m + memory: 512Mi diff --git a/helm/cas-estimation-tool/templates/route.yaml b/helm/cas-estimation-tool/templates/route.yaml new file mode 100644 index 0000000..491038e --- /dev/null +++ b/helm/cas-estimation-tool/templates/route.yaml @@ -0,0 +1,20 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ template "cas-estimation-tool.fullname" . }} + labels: +{{ include "cas-estimation-tool.labels" . | nindent 4 }} + annotations: + haproxy.router.openshift.io/balance: roundrobin +spec: + host: cas-estimation-tool.apps.silver.devops.gov.bc.ca + port: + targetPort: {{ template "cas-estimation-tool.fullname" . }} + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + to: + kind: Service + name: {{ template "cas-estimation-tool.fullname" . }} + weight: 100 + wildcardPolicy: None diff --git a/helm/cas-estimation-tool/templates/service.yaml b/helm/cas-estimation-tool/templates/service.yaml new file mode 100644 index 0000000..0797042 --- /dev/null +++ b/helm/cas-estimation-tool/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "cas-estimation-tool.fullname" . }} + labels: {{ include "cas-estimation-tool.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 8000 + protocol: TCP + name: {{ template "cas-estimation-tool.fullname" . }} + selector: {{ include "cas-estimation-tool.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/helm/cas-estimation-tool/values.yaml b/helm/cas-estimation-tool/values.yaml new file mode 100644 index 0000000..f7e7a54 --- /dev/null +++ b/helm/cas-estimation-tool/values.yaml @@ -0,0 +1 @@ +databaseSecretName: cas-estimation-db-cas-postgres-cluster-pguser-estimation diff --git a/helm/database/values.yaml b/helm/database/values.yaml index 210e840..afc0c22 100644 --- a/helm/database/values.yaml +++ b/helm/database/values.yaml @@ -1,5 +1,5 @@ postgresCluster: - storageSize: 512Mi + storageSize: 400Mi postgres: replicaCount: 2 pgbouncer: From fbb3546e76dc423ae1c4dc21b3a59a919db933a6 Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 13:07:57 -0700 Subject: [PATCH 4/7] chore: postgres connection --- .../templates/deployment.yaml | 2 +- .../templates/network_policy.yaml | 17 +++++++++++++++++ helm/cas-estimation-tool/values.yaml | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 helm/cas-estimation-tool/templates/network_policy.yaml diff --git a/helm/cas-estimation-tool/templates/deployment.yaml b/helm/cas-estimation-tool/templates/deployment.yaml index 089b8a4..1794466 100644 --- a/helm/cas-estimation-tool/templates/deployment.yaml +++ b/helm/cas-estimation-tool/templates/deployment.yaml @@ -57,7 +57,7 @@ spec: - containerPort: 8000 resources: limits: - cpu: 500m + cpu: 250m memory: 1Gi requests: cpu: 25m diff --git a/helm/cas-estimation-tool/templates/network_policy.yaml b/helm/cas-estimation-tool/templates/network_policy.yaml new file mode 100644 index 0000000..50b7a3e --- /dev/null +++ b/helm/cas-estimation-tool/templates/network_policy.yaml @@ -0,0 +1,17 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ include "cas-estimation-tool.fullname" . }}-db-access + labels: {{ include "cas-estimation-tool.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: cas-estimation-db + ingress: + - from: + - podSelector: + matchLabels: + release: {{ include "cas-estimation-tool.name" . }} + - podSelector: + matchLabels: + app.kubernetes.io/instance: {{ include "cas-estimation-tool.name" . }} \ No newline at end of file diff --git a/helm/cas-estimation-tool/values.yaml b/helm/cas-estimation-tool/values.yaml index f7e7a54..ecd684e 100644 --- a/helm/cas-estimation-tool/values.yaml +++ b/helm/cas-estimation-tool/values.yaml @@ -1 +1 @@ -databaseSecretName: cas-estimation-db-cas-postgres-cluster-pguser-estimation +databaseSecretName: cas-estimation-db-cas-postgres-cluster-pguser-postgres From 55e88eb1e0dce76df159ea3f650f94684b4be52f Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 13:37:26 -0700 Subject: [PATCH 5/7] chore: allowed_hosts --- estimation/estimation/settings.py | 7 +++++- .../templates/network_policy.yaml | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/estimation/estimation/settings.py b/estimation/estimation/settings.py index 367206d..58e79f3 100644 --- a/estimation/estimation/settings.py +++ b/estimation/estimation/settings.py @@ -29,7 +29,12 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1"] +ALLOWED_HOSTS = [ + "0.0.0.0", + "localhost", + "127.0.0.1", + "cas-estimation-tool.apps.silver.devops.gov.bc.ca", +] # Application definition diff --git a/helm/cas-estimation-tool/templates/network_policy.yaml b/helm/cas-estimation-tool/templates/network_policy.yaml index 50b7a3e..497c0ea 100644 --- a/helm/cas-estimation-tool/templates/network_policy.yaml +++ b/helm/cas-estimation-tool/templates/network_policy.yaml @@ -14,4 +14,24 @@ spec: release: {{ include "cas-estimation-tool.name" . }} - podSelector: matchLabels: - app.kubernetes.io/instance: {{ include "cas-estimation-tool.name" . }} \ No newline at end of file + app.kubernetes.io/instance: {{ include "cas-estimation-tool.name" . }} + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "cas-estimation-tool.fullname" . }}-allow-from-openshift-ingress +spec: + # This policy allows any pod with a route & service combination + # to accept traffic from the OpenShift router pods. This is + # required for things outside of OpenShift (like the Internet) + # to reach your pods. + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: ingress + podSelector: {} + policyTypes: + - Ingress + \ No newline at end of file From 2755d5120e530e351d023e89a6554069cefbdfa2 Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 13:50:37 -0700 Subject: [PATCH 6/7] chore: redirect_url --- helm/cas-estimation-tool/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/cas-estimation-tool/templates/deployment.yaml b/helm/cas-estimation-tool/templates/deployment.yaml index 1794466..f0258d7 100644 --- a/helm/cas-estimation-tool/templates/deployment.yaml +++ b/helm/cas-estimation-tool/templates/deployment.yaml @@ -28,6 +28,8 @@ spec: secretKeyRef: name: "cas-estimation-tool-github" key: client_secret + - name: GITHUB_REDIRECT_URI + value: https://console.apps.silver.devops.gov.bc.ca/github/callback/ - name: TEAM_MEMBERS valueFrom: secretKeyRef: From 614721526b8163eb4dd5616802815ae36a92f2b0 Mon Sep 17 00:00:00 2001 From: Pierre Bastianelli Date: Thu, 19 Sep 2024 14:46:52 -0700 Subject: [PATCH 7/7] chore: csrf --- estimation/estimation/settings.py | 2 ++ helm/cas-estimation-tool/templates/deployment.yaml | 2 +- helm/deploy_db.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/estimation/estimation/settings.py b/estimation/estimation/settings.py index 58e79f3..ed17520 100644 --- a/estimation/estimation/settings.py +++ b/estimation/estimation/settings.py @@ -36,6 +36,8 @@ "cas-estimation-tool.apps.silver.devops.gov.bc.ca", ] +CSRF_TRUSTED_ORIGINS = ["https://cas-estimation-tool.apps.silver.devops.gov.bc.ca"] + # Application definition diff --git a/helm/cas-estimation-tool/templates/deployment.yaml b/helm/cas-estimation-tool/templates/deployment.yaml index f0258d7..04cc868 100644 --- a/helm/cas-estimation-tool/templates/deployment.yaml +++ b/helm/cas-estimation-tool/templates/deployment.yaml @@ -29,7 +29,7 @@ spec: name: "cas-estimation-tool-github" key: client_secret - name: GITHUB_REDIRECT_URI - value: https://console.apps.silver.devops.gov.bc.ca/github/callback/ + value: https://cas-estimation-tool.apps.silver.devops.gov.bc.ca/github/callback/ - name: TEAM_MEMBERS valueFrom: secretKeyRef: diff --git a/helm/deploy_db.sh b/helm/deploy_db.sh index f8e8bc5..c8defdf 100755 --- a/helm/deploy_db.sh +++ b/helm/deploy_db.sh @@ -7,6 +7,6 @@ helm repo add cas-postgres https://bcgov.github.io/cas-postgres/ helm repo update helm upgrade --install --atomic --timeout 1800s \ - --namespace "cf52af-test" \ + --namespace "599f0a-dev" \ --values ./database/values.yaml \ cas-estimation-db cas-postgres/cas-postgres-cluster --version 1.1.1