feat: add deployment/IaC for nginx

bcgov · Nov 14, 2024 · a257853 · a257853
1 parent 1546eac
commit a257853
Show file tree

Hide file tree

Showing 3 changed files with 258 additions and 2 deletions.
diff --git a/.github/workflows/deploy-to-openshift-backend-dev.yml b/.github/workflows/deploy-to-openshift-backend-dev.yml
@@ -34,6 +34,13 @@ env:
   MAX_CPU: "250m"
   MIN_MEM: "200Mi"
   MAX_MEM: "700Mi"
+
+  MIN_REPLICAS_NGINX: "1"
+  MAX_REPLICAS_NGINX: "2"
+  MIN_CPU_NGINX: "50m"
+  MAX_CPU_NGINX: "100m"
+  MIN_MEM_NGINX: "200Mi"
+  MAX_MEM_NGINX: "250Mi"
   # SITE_URL should have no scheme or port. It will be prepended with https://
   HOST_ROUTE: ${{ secrets.SITE_URL }}
 
@@ -143,6 +150,9 @@ jobs:
           oc rollout cancel deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }} 2> /dev/null \
               || true && echo "No rollout in progress"
 
+          oc rollout cancel deployment/${{ env.APP_NAME }}-nginx-${{ env.APP_ENVIRONMENT }} 2> /dev/null \
+              || true && echo "No rollout in progress"
+
           oc tag \
              ${{ steps.push-image-backend.outputs.registry-path }} \
              ${{ env.REPO_NAME }}-${{ env.IMAGE_NAME }}-${{ env.BRANCH }}:${{ env.TAG }}
@@ -165,6 +175,28 @@ jobs:
              -p APP_ENVIRONMENT=${{ env.APP_ENVIRONMENT }} \
               | oc apply -f -
 
+          # Process and apply nginx deployment template
+          oc process \
+             -f tools/openshift/nginx.deployment.yaml \
+             -p APP_NAME=${{ env.APP_NAME }} \
+             -p REPO_NAME=${{ env.REPO_NAME }} \
+             -p BRANCH=${{ env.BRANCH }} \
+             -p NAMESPACE=${{ env.OPENSHIFT_NAMESPACE }} \
+             -p MIN_REPLICAS=${{ env.MIN_REPLICAS_NGINX }} \
+             -p MAX_REPLICAS=${{ env.MAX_REPLICAS_NGINX }} \
+             -p MIN_CPU=${{ env.MIN_CPU_NGINX }} \
+             -p MAX_CPU=${{ env.MAX_CPU_NGINX }} \
+             -p MIN_MEM=${{ env.MIN_MEM_NGINX }} \
+             -p MAX_MEM=${{ env.MAX_MEM_NGINX }} \
+             -p HOST_ROUTE=${{ secrets.NGINX_HOST_ROUTE }} \
+             -p APP_ENVIRONMENT=${{ env.APP_ENVIRONMENT }} \
+             -p NGINX_IMAGE=${{ vars.NGINX_IMAGE }}
+              | oc apply -f -
+
+          cat <<- EOF | sed 's/^ \+//g' > /tmp/nginx_htpasswd
+            ${{ secrets.D365_BC_REGISTRY_API }}
+          EOF
+
           # Process update-configmap
           curl -s https://raw.githubusercontent.com/bcgov/${{ env.REPO_NAME }}/${{ env.BRANCH }}/tools/config/update-configmap.sh \
             | bash /dev/stdin \
@@ -178,15 +210,23 @@ jobs:
               ${{ secrets.SOAM_CLIENT_SECRET_IDIR }} \
               ${{ secrets.SPLUNK_TOKEN }} \
               ${{ secrets.REDIS_PASSWORD }} \
-              ${{ secrets.D365_API_PREFIX }}
+              ${{ secrets.D365_API_PREFIX }} \
+              /tmp/nginx_htpasswd
 
-          # Start rollout (if necessary) and follow it
+          # Start rollout of the application
           oc rollout restart deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }} 2> /dev/null \
               || true && echo "Rollout in progress"
 
           # Get status, returns 0 if rollout is successful
           oc rollout status deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }}
 
+          # Start rollout of nginx
+          oc rollout restart deployment/${{ env.APP_NAME }}-nginx-${{ env.APP_ENVIRONMENT }} 2> /dev/null \
+              || true && echo "Rollout in progress"
+
+          # Get status, returns 0 if rollout is successful
+          oc rollout status deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }}
+
       - name: ZAP Scan
         uses: zaproxy/[email protected]
         with:

diff --git a/tools/config/update-configmap.sh b/tools/config/update-configmap.sh
@@ -11,6 +11,7 @@ readonly SOAM_CLIENT_SECRET_IDIR=$8
 readonly SPLUNK_TOKEN=$9
 readonly REDIS_PASSWORD=${10}
 readonly D365_API_PREFIX=${11}
+readonly NGINX_HTPASSWD_FILE=${12:-unset}
 readonly SOAM_KC_REALM_ID="standard"
 readonly D365_API_ENDPOINT="http://$D365_API_PREFIX-$ENV_VAL:5091"
 readonly TIMEZONE="America/Vancouver"
@@ -113,6 +114,50 @@ oc create -n "$OPENSHIFT_NAMESPACE" configmap \
   --from-literal="ISSUER=EDUC_CCOF" \
   --dry-run -o yaml | oc apply -f -
 
+if [ "$ENV_VAL" = 'dev' ]; then
+  API_CLUSTER_ADDRESS="http://$D365_API_PREFIX-$ENV_VAL.$OPENSHIFT_NAMESPACE.svc.cluster.local:5091"
+
+  NGINX_CONF << CONF
+server {
+  listen 8080 default_server;
+  location / {
+      proxy_ignore_client_abort on;
+      proxy_pass $API_CLUSTER_ADDRESS;
+      proxy_redirect off;
+      proxy_set_header  X-Real-IP  \$remote_addr;
+      proxy_set_header  X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header  Host \$http_host;
+      expires -1;
+      proxy_no_cache 1;
+      auth_basic "Authentication Required";
+      auth_basic_user_file /etc/nginx/conf.d/htpasswd;
+  }
+  location /api/Health {
+      proxy_ignore_client_abort on;
+      proxy_pass $API_CLUSTER_ADDRESS/api/Health;
+      proxy_redirect off;
+      proxy_set_header  X-Real-IP  \$remote_addr;
+      proxy_set_header  X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header  Host \$http_host;
+      expires -1;
+      proxy_no_cache 1;
+  }
+}
+CONF
+
+  if [ "$NGINX_HTPASSWD_FILE" != 'unset' ]; then
+    echo Creating config map "$APP_NAME-nginx-config-map"
+    oc create -n "$OPENSHIFT_NAMESPACE" \
+      configmap "$APP_NAME-nginx-config-map" \
+      --from-literal=default.conf="$NGINX_CONF" \
+      --from-file=htpasswd="$NGINX_HTPASSWD_FILE" \
+      --dry-run -o yaml | oc apply -f -
+  else
+    echo "Nginx htpasswd file unset, cannot create nginx config map!"
+    exit 1
+  fi
+fi
+
 if [ "$ENV_VAL" != 'qa' ]; then
     SPLUNK_URL="gww.splunk.educ.gov.bc.ca"
     FLB_CONFIG="[SERVICE]
@@ -157,3 +202,4 @@ if [ "$ENV_VAL" != 'qa' ]; then
        --from-literal=parsers.conf="$PARSER_CONFIG" \
        --dry-run -o yaml | oc apply -f -
 fi
+
diff --git a/tools/openshift/nginx.deployment.yaml b/tools/openshift/nginx.deployment.yaml
@@ -0,0 +1,170 @@
+---
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: ${REPO_NAME}-nginx-deployment
+  labels:
+    template: ${REPO_NAME}-template
+objects:
+- apiVersion: apps/v1
+  kind: Deployment
+  metadata:
+    annotations:
+      openshift.io/generated-by: OpenShiftNewApp
+    labels:
+      app: "${APP_NAME}-${BRANCH}"
+      branch: "${BRANCH}"
+    name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}"
+  spec:
+    replicas: ${{MIN_REPLICAS}}
+    selector:
+      matchLabels:
+        app: "${APP_NAME}-${BRANCH}"
+    strategy:
+      type: RollingUpdate
+      rollingUpdate:
+        maxUnavailable: 25%
+        maxSurge: 25%
+    template:
+      metadata:
+        annotations:
+          openshift.io/generated-by: OpenShiftNewApp
+        labels:
+          app: "${APP_NAME}-${BRANCH}"
+      spec:
+        containers:
+        - image: artifacts.developer.gov.bc.ca/docker-remote/${NGINX_IMAGE}
+          imagePullPolicy: Always
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /
+              port: 80
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}"
+          ports:
+          - containerPort: 80
+            protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /
+              port: 80
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          resources:
+            requests:
+              cpu: "${MIN_CPU}"
+              memory: "${MIN_MEM}"
+            limits:
+              cpu: "${MAX_CPU}"
+              memory: "${MAX_MEM}"
+          volumeMounts:
+          - mountPath: /etc/tls-certs
+            name: tls-certs
+            readOnly: true
+          - mountPath: /etc/nginx/conf.d
+            name: config-env
+        volumes:
+        - name: tls-certs
+          secret:
+            secretName: ccof-frontend-cert
+        - configMap:
+            name: ${APP_NAME}-nginx-config-map
+          name: config-env
+    test: false
+- apiVersion: v1
+  kind: Service
+  metadata:
+    annotations:
+      openshift.io/generated-by: OpenShiftNewApp
+      service.alpha.openshift.io/serving-cert-secret-name: ccof-frontend-cert
+    labels:
+      app: "${APP_NAME}-${BRANCH}"
+    name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}"
+  spec:
+    ports:
+    - name: 80-tcp
+      port: 80
+      protocol: TCP
+      targetPort: 80
+    selector:
+      app: ccof-app
+- apiVersion: v1
+  kind: Route
+  metadata:
+    annotations:
+      openshift.io/host.generated: "true"
+    labels:
+      app: "${APP_NAME}-${BRANCH}"
+    name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}"
+  spec:
+    host: "${HOST_ROUTE}"
+    port:
+      targetPort: 80-tcp
+    tls:
+      insecureEdgeTerminationPolicy: Redirect
+      termination: edge
+    to:
+      kind: Service
+      name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}"
+      weight: 100
+    wildcardPolicy: None
+- apiVersion: autoscaling/v2
+  kind: HorizontalPodAutoscaler
+  metadata:
+    name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}-cpu-autoscaler"
+  spec:
+    minReplicas: ${{MIN_REPLICAS}}
+    maxReplicas: ${{MAX_REPLICAS}}
+    metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        targetAverageUtilization: 90
+    scaleTargetRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: ${APP_NAME}-nginx-${APP_ENVIRONMENT}
+parameters:
+- name: REPO_NAME
+  description: Application repository name
+  required: true
+- name: BRANCH
+  description: Job identifier (i.e. 'pr-5' OR 'master')
+  required: true
+- name: NAMESPACE
+  description: Target namespace reference (i.e. 'k8vopl-dev')
+  required: true
+- name: APP_NAME
+  description: Application name
+  required: true
+- name: MIN_REPLICAS
+  description: The minimum amount of replicas
+  required: true
+- name: MAX_REPLICAS
+  description: The maximum amount of replicas
+  required: true
+- name: MIN_CPU
+  description: The minimum amount of cpu
+  required: true
+- name: MAX_CPU
+  description: The maximum amount of cpu
+  required: true
+- name: MIN_MEM
+  description: The minimum amount of memory
+  required: true
+- name: MAX_MEM
+  description: The maximum amount of memory
+  required: true
+- name: APP_ENVIRONMENT
+  description: The environment being created ('dev', 'qa', 'uat', 'prod')
+  required: true
+- name: NGINX_IMAGE
+  description: The nginx docker image to use
+  required: true