From a2578532b2d0b1bfe32270bbd05b5417ef8ed306 Mon Sep 17 00:00:00 2001 From: Trevor Richards Date: Wed, 13 Nov 2024 19:27:50 -0800 Subject: [PATCH] feat: add deployment/IaC for nginx --- .../deploy-to-openshift-backend-dev.yml | 44 ++++- tools/config/update-configmap.sh | 46 +++++ tools/openshift/nginx.deployment.yaml | 170 ++++++++++++++++++ 3 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 tools/openshift/nginx.deployment.yaml diff --git a/.github/workflows/deploy-to-openshift-backend-dev.yml b/.github/workflows/deploy-to-openshift-backend-dev.yml index 1b0c61136..4a572f9b9 100644 --- a/.github/workflows/deploy-to-openshift-backend-dev.yml +++ b/.github/workflows/deploy-to-openshift-backend-dev.yml @@ -34,6 +34,13 @@ env: MAX_CPU: "250m" MIN_MEM: "200Mi" MAX_MEM: "700Mi" + + MIN_REPLICAS_NGINX: "1" + MAX_REPLICAS_NGINX: "2" + MIN_CPU_NGINX: "50m" + MAX_CPU_NGINX: "100m" + MIN_MEM_NGINX: "200Mi" + MAX_MEM_NGINX: "250Mi" # SITE_URL should have no scheme or port. It will be prepended with https:// HOST_ROUTE: ${{ secrets.SITE_URL }} @@ -143,6 +150,9 @@ jobs: oc rollout cancel deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }} 2> /dev/null \ || true && echo "No rollout in progress" + oc rollout cancel deployment/${{ env.APP_NAME }}-nginx-${{ env.APP_ENVIRONMENT }} 2> /dev/null \ + || true && echo "No rollout in progress" + oc tag \ ${{ steps.push-image-backend.outputs.registry-path }} \ ${{ env.REPO_NAME }}-${{ env.IMAGE_NAME }}-${{ env.BRANCH }}:${{ env.TAG }} @@ -165,6 +175,28 @@ jobs: -p APP_ENVIRONMENT=${{ env.APP_ENVIRONMENT }} \ | oc apply -f - + # Process and apply nginx deployment template + oc process \ + -f tools/openshift/nginx.deployment.yaml \ + -p APP_NAME=${{ env.APP_NAME }} \ + -p REPO_NAME=${{ env.REPO_NAME }} \ + -p BRANCH=${{ env.BRANCH }} \ + -p NAMESPACE=${{ env.OPENSHIFT_NAMESPACE }} \ + -p MIN_REPLICAS=${{ env.MIN_REPLICAS_NGINX }} \ + -p MAX_REPLICAS=${{ env.MAX_REPLICAS_NGINX }} \ + -p MIN_CPU=${{ env.MIN_CPU_NGINX }} \ + -p MAX_CPU=${{ env.MAX_CPU_NGINX }} \ + -p MIN_MEM=${{ env.MIN_MEM_NGINX }} \ + -p MAX_MEM=${{ env.MAX_MEM_NGINX }} \ + -p HOST_ROUTE=${{ secrets.NGINX_HOST_ROUTE }} \ + -p APP_ENVIRONMENT=${{ env.APP_ENVIRONMENT }} \ + -p NGINX_IMAGE=${{ vars.NGINX_IMAGE }} + | oc apply -f - + + cat <<- EOF | sed 's/^ \+//g' > /tmp/nginx_htpasswd + ${{ secrets.D365_BC_REGISTRY_API }} + EOF + # Process update-configmap curl -s https://raw.githubusercontent.com/bcgov/${{ env.REPO_NAME }}/${{ env.BRANCH }}/tools/config/update-configmap.sh \ | bash /dev/stdin \ @@ -178,15 +210,23 @@ jobs: ${{ secrets.SOAM_CLIENT_SECRET_IDIR }} \ ${{ secrets.SPLUNK_TOKEN }} \ ${{ secrets.REDIS_PASSWORD }} \ - ${{ secrets.D365_API_PREFIX }} + ${{ secrets.D365_API_PREFIX }} \ + /tmp/nginx_htpasswd - # Start rollout (if necessary) and follow it + # Start rollout of the application oc rollout restart deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }} 2> /dev/null \ || true && echo "Rollout in progress" # Get status, returns 0 if rollout is successful oc rollout status deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }} + # Start rollout of nginx + oc rollout restart deployment/${{ env.APP_NAME }}-nginx-${{ env.APP_ENVIRONMENT }} 2> /dev/null \ + || true && echo "Rollout in progress" + + # Get status, returns 0 if rollout is successful + oc rollout status deployment/${{ env.APP_NAME }}-${{ env.IMAGE_NAME }}-${{ env.APP_ENVIRONMENT }} + - name: ZAP Scan uses: zaproxy/action-full-scan@v0.8.0 with: diff --git a/tools/config/update-configmap.sh b/tools/config/update-configmap.sh index 22af234fd..9bb06899a 100644 --- a/tools/config/update-configmap.sh +++ b/tools/config/update-configmap.sh @@ -11,6 +11,7 @@ readonly SOAM_CLIENT_SECRET_IDIR=$8 readonly SPLUNK_TOKEN=$9 readonly REDIS_PASSWORD=${10} readonly D365_API_PREFIX=${11} +readonly NGINX_HTPASSWD_FILE=${12:-unset} readonly SOAM_KC_REALM_ID="standard" readonly D365_API_ENDPOINT="http://$D365_API_PREFIX-$ENV_VAL:5091" readonly TIMEZONE="America/Vancouver" @@ -113,6 +114,50 @@ oc create -n "$OPENSHIFT_NAMESPACE" configmap \ --from-literal="ISSUER=EDUC_CCOF" \ --dry-run -o yaml | oc apply -f - +if [ "$ENV_VAL" = 'dev' ]; then + API_CLUSTER_ADDRESS="http://$D365_API_PREFIX-$ENV_VAL.$OPENSHIFT_NAMESPACE.svc.cluster.local:5091" + + NGINX_CONF << CONF +server { + listen 8080 default_server; + location / { + proxy_ignore_client_abort on; + proxy_pass $API_CLUSTER_ADDRESS; + proxy_redirect off; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header Host \$http_host; + expires -1; + proxy_no_cache 1; + auth_basic "Authentication Required"; + auth_basic_user_file /etc/nginx/conf.d/htpasswd; + } + location /api/Health { + proxy_ignore_client_abort on; + proxy_pass $API_CLUSTER_ADDRESS/api/Health; + proxy_redirect off; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header Host \$http_host; + expires -1; + proxy_no_cache 1; + } +} +CONF + + if [ "$NGINX_HTPASSWD_FILE" != 'unset' ]; then + echo Creating config map "$APP_NAME-nginx-config-map" + oc create -n "$OPENSHIFT_NAMESPACE" \ + configmap "$APP_NAME-nginx-config-map" \ + --from-literal=default.conf="$NGINX_CONF" \ + --from-file=htpasswd="$NGINX_HTPASSWD_FILE" \ + --dry-run -o yaml | oc apply -f - + else + echo "Nginx htpasswd file unset, cannot create nginx config map!" + exit 1 + fi +fi + if [ "$ENV_VAL" != 'qa' ]; then SPLUNK_URL="gww.splunk.educ.gov.bc.ca" FLB_CONFIG="[SERVICE] @@ -157,3 +202,4 @@ if [ "$ENV_VAL" != 'qa' ]; then --from-literal=parsers.conf="$PARSER_CONFIG" \ --dry-run -o yaml | oc apply -f - fi + diff --git a/tools/openshift/nginx.deployment.yaml b/tools/openshift/nginx.deployment.yaml new file mode 100644 index 000000000..a560ac0f3 --- /dev/null +++ b/tools/openshift/nginx.deployment.yaml @@ -0,0 +1,170 @@ +--- +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ${REPO_NAME}-nginx-deployment + labels: + template: ${REPO_NAME}-template +objects: +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + openshift.io/generated-by: OpenShiftNewApp + labels: + app: "${APP_NAME}-${BRANCH}" + branch: "${BRANCH}" + name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}" + spec: + replicas: ${{MIN_REPLICAS}} + selector: + matchLabels: + app: "${APP_NAME}-${BRANCH}" + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + template: + metadata: + annotations: + openshift.io/generated-by: OpenShiftNewApp + labels: + app: "${APP_NAME}-${BRANCH}" + spec: + containers: + - image: artifacts.developer.gov.bc.ca/docker-remote/${NGINX_IMAGE} + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 80 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}" + ports: + - containerPort: 80 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: / + port: 80 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: "${MIN_CPU}" + memory: "${MIN_MEM}" + limits: + cpu: "${MAX_CPU}" + memory: "${MAX_MEM}" + volumeMounts: + - mountPath: /etc/tls-certs + name: tls-certs + readOnly: true + - mountPath: /etc/nginx/conf.d + name: config-env + volumes: + - name: tls-certs + secret: + secretName: ccof-frontend-cert + - configMap: + name: ${APP_NAME}-nginx-config-map + name: config-env + test: false +- apiVersion: v1 + kind: Service + metadata: + annotations: + openshift.io/generated-by: OpenShiftNewApp + service.alpha.openshift.io/serving-cert-secret-name: ccof-frontend-cert + labels: + app: "${APP_NAME}-${BRANCH}" + name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}" + spec: + ports: + - name: 80-tcp + port: 80 + protocol: TCP + targetPort: 80 + selector: + app: ccof-app +- apiVersion: v1 + kind: Route + metadata: + annotations: + openshift.io/host.generated: "true" + labels: + app: "${APP_NAME}-${BRANCH}" + name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}" + spec: + host: "${HOST_ROUTE}" + port: + targetPort: 80-tcp + tls: + insecureEdgeTerminationPolicy: Redirect + termination: edge + to: + kind: Service + name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}" + weight: 100 + wildcardPolicy: None +- apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + name: "${APP_NAME}-nginx-${APP_ENVIRONMENT}-cpu-autoscaler" + spec: + minReplicas: ${{MIN_REPLICAS}} + maxReplicas: ${{MAX_REPLICAS}} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 90 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ${APP_NAME}-nginx-${APP_ENVIRONMENT} +parameters: +- name: REPO_NAME + description: Application repository name + required: true +- name: BRANCH + description: Job identifier (i.e. 'pr-5' OR 'master') + required: true +- name: NAMESPACE + description: Target namespace reference (i.e. 'k8vopl-dev') + required: true +- name: APP_NAME + description: Application name + required: true +- name: MIN_REPLICAS + description: The minimum amount of replicas + required: true +- name: MAX_REPLICAS + description: The maximum amount of replicas + required: true +- name: MIN_CPU + description: The minimum amount of cpu + required: true +- name: MAX_CPU + description: The maximum amount of cpu + required: true +- name: MIN_MEM + description: The minimum amount of memory + required: true +- name: MAX_MEM + description: The maximum amount of memory + required: true +- name: APP_ENVIRONMENT + description: The environment being created ('dev', 'qa', 'uat', 'prod') + required: true +- name: NGINX_IMAGE + description: The nginx docker image to use + required: true