Merge pull request #5 from bcgov/main

add vault_example DAG
bcgov · Apr 9, 2024 · c3e068e · c3e068e
2 parents 1562d7b + 3a3cafa
commit c3e068e
Show file tree

Hide file tree

Showing 39 changed files with 284 additions and 971 deletions.
diff --git a/README.md b/README.md
@@ -1,29 +1,28 @@
-# Airflow Set Up
+# Airflow Set Up for NRM Data Analytics Platform
 
 ## Helm chart sourced from
 ```sh
 helm pull oci://registry-1.docker.io/bitnamicharts/airflow
 ```
 
 ## Deploying to OpenShift
+Create OpenShift ConfigMap for requirements.txt:
 ```sh
-helm install test-release-af .
+oc create configmap airflow-requirements --from-file=requirements.txt
+```
+Navigate to the 'oc' folder then:
+```sh
+oc apply -f .
+```
+Navigate to the 'airflow' folder then:
+```sh
+helm install airflow .
 ```
 
 ## Visit the application here:
-http://nr-airflow-dev.apps.emerald.devops.gov.bc.ca/
+http://nr-airflow.apps.emerald.devops.gov.bc.ca/
 
 ## Upgrade OpenShift Deployment
 ```sh 
-helm upgrade -f values.yaml test-release-af . --version 16.1.2  
-```
-
-## Create OpenShift ConfigMap for requirements.txt: 
-```sh
-oc create configmap airflow-requirements --from-file=requirements.txt
+helm upgrade -f values.yaml airflow .
 ```
-Delete if already exists
-
-
-More info: https://apps.nrs.gov.bc.ca/int/confluence/x/zQ09Cg
-
diff --git a/airflow/charts/postgresql/values.yaml b/airflow/charts/postgresql/values.yaml
@@ -65,7 +65,7 @@ extraDeploy: []
 ##
 commonLabels:
   DataClass: Medium # Abi: 'Medium' for now, need to look deeper into the metadata being stored
-  Release: test-release-af # Abi: Release label is attached to nwp
+  Release: airflow # Abi: Release label is attached to nwp
 ## @param commonAnnotations Add annotations to all the deployed resources
 ##
 commonAnnotations: {}
@@ -683,7 +683,7 @@ primary:
       - ReadWriteOnce
     ## @param primary.persistence.size PVC Storage Request for PostgreSQL volume
     ##
-    size: 4Gi # Abi: lowered for dev and test environments
+    size: 2Gi # Abi: lowered
     ## @param primary.persistence.annotations Annotations for the PVC
     ##
     annotations: {}
@@ -1047,7 +1047,7 @@ readReplicas:
       - ReadWriteOnce
     ## @param readReplicas.persistence.size PVC Storage Request for PostgreSQL volume
     ##
-    size: 8Gi
+    size: 2Gi # Abi: previously 8Gi
     ## @param readReplicas.persistence.annotations Annotations for the PVC
     ##
     annotations: {}
@@ -1164,7 +1164,7 @@ backup:
        - ReadWriteOnce
       ## @param backup.cronjob.storage.size PVC Storage Request for the backup data volume
       ##
-      size: 8Gi
+      size: 2Gi # Abi: previously 8Gi
       ## @param backup.cronjob.storage.annotations PVC annotations
       ##
       annotations: {}

diff --git a/airflow/values.yaml b/airflow/values.yaml
@@ -41,8 +41,8 @@ extraDeploy: []
 ## @param commonLabels Add labels to all the deployed resources
 ##
 commonLabels: 
-  DataClass: Low # Abi: Not the same as the DataClass for the jobs. 'Low' for app pods, usually 'Medium' or 'High' for jobs.
-  Release: test-release-af # Abi: Release label is attached to nwp
+  DataClass: Low # Abi: Not the same as the DataClass for the jobs. 'Low' for app pods, usually 'Medium' for jobs.
+  Release: airflow # Abi: Release label is attached to nwp
   # Internet-Ingress: ALLOW
 
 ## @param commonAnnotations Add annotations to all the deployed resources
@@ -328,10 +328,10 @@ web:
   resources: #Abi: keep requests low but be careful with limits as it crashes when memory limit is hit
     requests: 
       memory: 600Mi
-      cpu: 10m
+      cpu: 50m
     limits: 
       memory: 2Gi
-      cpu: '2' 
+      cpu: 1000m
   ## Configure Airflow web pods Security Context
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   ## @param web.podSecurityContext.enabled Enabled Airflow web pods' Security Context
@@ -556,7 +556,7 @@ scheduler:
       cpu: 10m
     limits: 
       memory: 1Gi
-      cpu: '1'
+      cpu: 1000m
   ## Configure Airflow scheduler pods Security Context
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
   ## @param scheduler.podSecurityContext.enabled Enabled Airflow scheduler pods' Security Context
@@ -1053,10 +1053,15 @@ git:
     #  git checkout 
     repositories:
       - repository: https://github.com/bcgov/nr-airflow
-        branch: a1b9b0-dev
+        branch: a1b9b0-prod
         name: nr-airflow-dags
         path: /dags
 
+      - repository: https://github.com/bcgov/nr-dap-dlh
+        branch: main
+        name: nr-dap-dlh-dags
+        path: /pmt/af/DAG/
+
       # - repository: ""
         ## Branch from repository to checkout
         ##
@@ -1091,7 +1096,6 @@ git:
     command: []
     args: []
     extraVolumeMounts: []
-    # extraEnvVars: []
     # Abi: values found in Emerald documentation here: https://digital.gov.bc.ca/cloud/services/private/internal-resources/emerald/
     extraEnvVars:
       - name: http_proxy
@@ -1123,11 +1127,10 @@ git:
   ## @param git.sync.resources Sync sidecar container resource requests and limits
   ##
   sync:
-    interval: 90 # Abi: can increase in prod 
+    interval: 30 # Abi: increased
     command: []
     args: []
     extraVolumeMounts: []
-    # extraEnvVars: []
     # Abi: values found in Emerald documentation here: https://digital.gov.bc.ca/cloud/services/private/internal-resources/emerald/
     extraEnvVars:
       - name: http_proxy

diff --git a/dags/airbyte_list_connections.py b/dags/airbyte_list_connections.py
diff --git a/dags/airbyte_trigger_job.py b/dags/airbyte_trigger_job.py
diff --git a/dags/connection_secrets_example.py b/dags/connection_secrets_example.py
diff --git a/dags/data_pipeline_dag_fta.py b/dags/data_pipeline_dag_fta.py
diff --git a/dags/data_pipeline_dag_rrs.py b/dags/data_pipeline_dag_rrs.py
diff --git a/dags/dbt_dlh_example.py b/dags/dbt_dlh_example.py
diff --git a/dags/dbt_example.py b/dags/dbt_example.py
diff --git a/dags/dbt_rrs_snapshot.py b/dags/dbt_rrs_snapshot.py