Merge pull request #238 from bcgov/dev

Release Dev to Test

.config/config.json

@@ -41,37 +41,55 @@
   },
   "sso": {
     "dev": {
-      "url": "https://dev.loginproxy.gov.bc.ca/auth",
-      "clientId": "bio-hub-browser-4230",
+      "host": "https://dev.loginproxy.gov.bc.ca/auth",
       "realm": "standard",
-      "integrationId": "4230",
-      "adminHost": "https://loginproxy.gov.bc.ca/auth",
-      "adminUserName": "biohub-svc-4466",
-      "apiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
-      "keycloakSecret": "keycloak-admin-password",
-      "keycloakSecretAdminPassword": "keycloak_admin_password"
+      "clientId": "bio-hub-browser-4230",
+      "keycloakSecret": "keycloak",
+      "serviceClient": {
+        "serviceClientName": "biohub-svc-4466",
+        "keycloakSecretServiceClientPasswordKey": "biohub_svc_client_password"
+      },
+      "cssApi": {
+        "cssApiTokenUrl": "https://loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/token",
+        "cssApiClientId": "service-account-team-1159-4197",
+        "cssApiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
+        "keycloakSecretCssApiSecretKey": "css_api_client_secret",
+        "cssApiEnvironment": "dev"
+      }
     },
     "test": {
-      "url": "https://test.loginproxy.gov.bc.ca/auth",
-      "clientId": "bio-hub-browser-4230",
+      "host": "https://test.loginproxy.gov.bc.ca/auth",
       "realm": "standard",
-      "integrationId": "4230",
-      "adminHost": "https://loginproxy.gov.bc.ca/auth",
-      "adminUserName": "biohub-svc-4466",
-      "apiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
-      "keycloakSecret": "keycloak-admin-password",
-      "keycloakSecretAdminPassword": "keycloak_admin_password"
+      "clientId": "bio-hub-browser-4230",
+      "keycloakSecret": "keycloak",
+      "serviceClient": {
+        "serviceClientName": "biohub-svc-4466",
+        "keycloakSecretServiceClientPasswordKey": "biohub_svc_client_password"
+      },
+      "cssApi": {
+        "cssApiTokenUrl": "https://loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/token",
+        "cssApiClientId": "service-account-team-1159-4197",
+        "cssApiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
+        "keycloakSecretCssApiSecretKey": "css_api_client_secret",
+        "cssApiEnvironment": "test"
+      }
     },
     "prod": {
-      "url": "https://loginproxy.gov.bc.ca/auth",
-      "clientId": "bio-hub-browser-4230",
+      "host": "https://loginproxy.gov.bc.ca/auth",
       "realm": "standard",
-      "integrationId": "4230",
-      "adminHost": "https://loginproxy.gov.bc.ca/auth",
-      "adminUserName": "biohub-svc-4466",
-      "apiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
-      "keycloakSecret": "keycloak-admin-password",
-      "keycloakSecretAdminPassword": "keycloak_admin_password"
+      "clientId": "bio-hub-browser-4230",
+      "keycloakSecret": "keycloak",
+      "serviceClient": {
+        "serviceClientName": "biohub-svc-4466",
+        "keycloakSecretServiceClientPasswordKey": "biohub_svc_client_password"
+      },
+      "cssApi": {
+        "cssApiTokenUrl": "https://loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/token",
+        "cssApiClientId": "service-account-team-1159-4197",
+        "cssApiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
+        "keycloakSecretCssApiSecretKey": "css_api_client_secret",
+        "cssApiEnvironment": "prod"
+      }
     }
   }
 }

.github/pull_request_template.md

@@ -1,50 +1,11 @@
-# Overview
+## Links to Jira Tickets
 
-## Links to Jira tickets
+- {Include a link to all applicable Jira tickets}
 
-- {List all applicable Jira tickets}
+## Description of Changes
 
-## Description of relevant changes
+- {List all relevant code changes. Include any changes to the business workflow that might not be obvious to the reviewers of this PR.}
 
-- {List all relevant changes, in particular anything that will help the reviewers test/verify this PR}
+## Testing Notes
 
-## PR Checklist
-
-A list of items that are good to consider when making any changes.
-
-_Note: this list is not exhaustive, and not all items are always applicable._
-
-### Code
-
-- [ ] New files/classes/functions have appropriately descriptive names and comment blocks to describe their use/behaviour
-- [ ] I have avoided duplicating code when possible, moving re-usable pieces into functions
-- [ ] I have avoided hard-coding values where possible and moved any re-usable constants to a constants file
-- [ ] My code is as flat as possible (avoids deeply nested if/else blocks, promise chains, etc)
-- [ ] My code changes account for null/undefined values and handle errors appropriately
-- [ ] My code uses types/interfaces to help describe values/parameters/etc, help ensure type safety, and improve readability
-
-### Style
-
-- [ ] My code follows the established style conventions
-- [ ] My code uses native material-ui components/icons/conventions when possible
-
-### Documentation
-
-- [ ] I have commented my code sufficiently, such that an unfamiliar developer could understand my code
-- [ ] I have added/updated README's and related documentation, as needed
-
-### Tests
-
-- [ ] I have added/updated unit tests for any code I've added/updated
-- [ ] I have added/updated the Postman requests/tests to account for any API endpoints I've added/updated
-
-### Linting/Formatting
-
-- [ ] I have run the linter and fixed any issues, as needed  
-       _See the `lint` commands in package.json_
-- [ ] I have run the formatter and fixed any issues, as needed  
-       _See the `format` commands in package.json_
-
-### SonarCloud
-
-- [ ] I have addressed all SonarCloud Bugs, Vulnerabilities, Security Hotspots, and Code Smells
+- {List any relevant testing considerations, necessary pre-reqs, and areas of the app to focus on. Specifically, include anything that will help the reviewers of this PR verify the code is functioning as expected.}

.github/workflows/cleanClosedPR.yml

@@ -86,5 +86,5 @@ jobs:
         env:
           POD_SELECTOR: biohub-platform
         run: |
-          oc --namespace a0ec71-dev   get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash
-          oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash
+          oc --namespace a0ec71-dev   get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash
+          oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash

.github/workflows/deploy.yml

@@ -5,6 +5,8 @@ name: PR-Based Deploy on OpenShift
 on:
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
+    branches-ignore:
+      - prod
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.number }}
@@ -41,9 +43,11 @@ jobs:
       - checkEnv
     outputs:
       paths_result: ${{ steps.skip_check.outputs.paths_result }}
-      # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message.
+      # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message OR the base branch
+      # is dev, test, or prod.
       # Used to disable duplicate action skipping, if needed.
-      ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') }}
+      ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') || 
+        github.head_ref == 'dev' || github.head_ref == 'test' || github.head_ref == 'prod' }}
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@v5

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "git.ignoreLimitWarning": true
-}
+  "git.ignoreLimitWarning": true
+}

Makefile

@@ -56,6 +56,14 @@ clean: ## Closes and cleans (removes) all project containers
 	@echo "==============================================="
 	@docker-compose -f docker-compose.yml down -v --rmi all --remove-orphans
 
+prune: ## Deletes ALL docker artifacts (even those not associated to this project)
+	@echo -n "Delete ALL docker artifacts? [y/n] " && read ans && [ $${ans:-n} = y ]
+	@echo "==============================================="
+	@echo "Make: prune - deleting all docker artifacts"
+	@echo "==============================================="
+	@docker system prune --all --volumes -f
+	@docker volume prune --all -f
+
 ## ------------------------------------------------------------------------------
 ## Build/Run Postgres DB Commands
 ## - Builds all of the BioHub postgres db projects (db, db_setup)
@@ -82,13 +90,15 @@ build-backend: ## Builds all backend containers
 	@echo "==============================================="
 	@echo "Make: build-backend - building backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api queue
+	@docker-compose -f docker-compose.yml build db db_setup api
+	# @docker-compose -f docker-compose.yml build db db_setup api queue
 
 run-backend: ## Runs all backend containers
 	@echo "==============================================="
 	@echo "Make: run-backend - running backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api queue
+	@docker-compose -f docker-compose.yml up -d db db_setup api
+	# @docker-compose -f docker-compose.yml up -d db db_setup api queue
 
 ## ------------------------------------------------------------------------------
 ## Build/Run Backend+Web Commands (backend + web frontend)
@@ -99,13 +109,15 @@ build-web: ## Builds all backend+web containers
 	@echo "==============================================="
 	@echo "Make: build-web - building web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api queue app
+	@docker-compose -f docker-compose.yml build db db_setup api app
+	# @docker-compose -f docker-compose.yml build db db_setup api queue app
 
 run-web: ## Runs all backend+web containers
 	@echo "==============================================="
 	@echo "Make: run-web - running web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api queue app
+	@docker-compose -f docker-compose.yml up -d db db_setup api app
+	# @docker-compose -f docker-compose.yml up -d db db_setup api queue app
 
 ## ------------------------------------------------------------------------------
 ## Commands to shell into the target container

api/.pipeline/config.js

@@ -62,9 +62,9 @@ const phases = {
     env: 'build',
     tz: config.timezone.api,
     branch: branch,
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '1250m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '3Gi'
   },
   dev: {
@@ -83,15 +83,17 @@ const phases = {
     elasticsearchURL: 'http://es01:9200',
     elasticsearchEmlIndex: 'eml',
     elasticsearchTaxonomyIndex: 'taxonomy_3.0.0',
+    itisSolrUrl: 'https://services.itis.gov',
     s3KeyPrefix: (isStaticDeployment && 'biohub') || `local/${deployChangeId}/biohub`,
     tz: config.timezone.api,
     sso: config.sso.dev,
-    logLevel: 'debug',
-    cpuRequest: '100m',
-    cpuLimit: '500m',
-    memoryRequest: '512Mi',
-    memoryLimit: '2Gi',
-    replicas: '1',
+    logLevel: (isStaticDeployment && 'info') || 'debug',
+    nodeOptions: '--max_old_space_size=2250', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
+    cpuLimit: '600m',
+    memoryRequest: '100Mi',
+    memoryLimit: '3Gi',
+    replicas: (isStaticDeployment && '1') || '1',
     replicasMax: (isStaticDeployment && '2') || '1'
   },
   test: {
@@ -110,16 +112,18 @@ const phases = {
     elasticsearchURL: 'http://es01.a0ec71-dev:9200', // TODO: Update to test instance (es is not yet deployed to test)
     elasticsearchEmlIndex: 'eml',
     elasticsearchTaxonomyIndex: 'taxonomy_3.0.0',
+    itisSolrUrl: 'https://services.itis.gov',
     s3KeyPrefix: 'biohub',
     tz: config.timezone.api,
     sso: config.sso.test,
     logLevel: 'info',
-    cpuRequest: '200m',
+    nodeOptions: '--max_old_space_size=2250', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '1000m',
-    memoryRequest: '512Mi',
-    memoryLimit: '2Gi',
+    memoryRequest: '100Mi',
+    memoryLimit: '3Gi',
     replicas: '2',
-    replicasMax: '3'
+    replicasMax: '4'
   },
   prod: {
     namespace: 'a0ec71-prod',
@@ -137,16 +141,18 @@ const phases = {
     elasticsearchURL: 'http://es01:9200',
     elasticsearchEmlIndex: 'eml',
     elasticsearchTaxonomyIndex: 'taxonomy_3.0.0',
+    itisSolrUrl: 'https://services.itis.gov',
     s3KeyPrefix: 'biohub',
     tz: config.timezone.api,
     sso: config.sso.prod,
-    logLevel: 'info',
-    cpuRequest: '200m',
+    logLevel: 'warn',
+    nodeOptions: '--max_old_space_size=2250', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '1000m',
-    memoryRequest: '512Mi',
-    memoryLimit: '2Gi',
+    memoryRequest: '100Mi',
+    memoryLimit: '3Gi',
     replicas: '2',
-    replicasMax: '3'
+    replicasMax: '4'
   }
 };
 

api/.pipeline/lib/api.deploy.js

@@ -31,22 +31,39 @@ const apiDeploy = async (settings) => {
         HOST: phases[phase].host,
         CHANGE_ID: phases.build.changeId || changeId,
         APP_HOST: phases[phase].appHost,
+        // Node
         NODE_ENV: phases[phase].env || 'dev',
+        NODE_OPTIONS: phases[phase].nodeOptions,
+        // Elastic Search
         ELASTICSEARCH_URL: phases[phase].elasticsearchURL,
         ELASTICSEARCH_EML_INDEX: phases[phase].elasticsearchEmlIndex,
         ELASTICSEARCH_TAXONOMY_INDEX: phases[phase].elasticsearchTaxonomyIndex,
+        // ITIS SOLR
+        ITIS_SOLR_URL: phases[phase].itisSolrUrl,
+        // S3 (Object Store)
         S3_KEY_PREFIX: phases[phase].s3KeyPrefix,
+        OBJECT_STORE_SECRETS: 'biohubbc-object-store',
+        // Database
         TZ: phases[phase].tz,
-        KEYCLOAK_ADMIN_USERNAME: phases[phase].sso.adminUserName,
-        KEYCLOAK_SECRET: phases[phase].sso.keycloakSecret,
-        KEYCLOAK_SECRET_ADMIN_PASSWORD: phases[phase].sso.keycloakSecretAdminPassword,
         DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
-        KEYCLOAK_HOST: phases[phase].sso.url,
-        KEYCLOAK_CLIENT_ID: phases[phase].sso.clientId,
+        // Keycloak
+        KEYCLOAK_HOST: phases[phase].sso.host,
         KEYCLOAK_REALM: phases[phase].sso.realm,
-        KEYCLOAK_INTEGRATION_ID: phases[phase].sso.integrationId,
-        KEYCLOAK_API_HOST: phases[phase].sso.apiHost,
+        KEYCLOAK_CLIENT_ID: phases[phase].sso.clientId,
+        // Keycloak secret
+        KEYCLOAK_SECRET: phases[phase].sso.keycloakSecret,
+        // Keycloak Service Client
+        KEYCLOAK_ADMIN_USERNAME: phases[phase].sso.serviceClient.serviceClientName,
+        KEYCLOAK_SECRET_ADMIN_PASSWORD_KEY: phases[phase].sso.serviceClient.keycloakSecretServiceClientPasswordKey,
+        // Keycloak CSS API
+        KEYCLOAK_API_TOKEN_URL: phases[phase].sso.cssApi.cssApiTokenUrl,
+        KEYCLOAK_API_CLIENT_ID: phases[phase].sso.cssApi.cssApiClientId,
+        KEYCLOAK_API_CLIENT_SECRET_KEY: phases[phase].sso.cssApi.keycloakSecretCssApiSecretKey,
+        KEYCLOAK_API_HOST: phases[phase].sso.cssApi.cssApiHost,
+        KEYCLOAK_API_ENVIRONMENT: phases[phase].sso.cssApi.cssApiEnvironment,
+        // Log Level
         LOG_LEVEL: phases[phase].logLevel || 'info',
+        // Openshift Resources
         CPU_REQUEST: phases[phase].cpuRequest,
         CPU_LIMIT: phases[phase].cpuLimit,
         MEMORY_REQUEST: phases[phase].memoryRequest,

api/.pipeline/lib/queue.deploy.js

@@ -31,23 +31,21 @@ const queueDeploy = async (settings) => {
         HOST: phases[phase].host,
         CHANGE_ID: phases.build.changeId || changeId,
         APP_HOST: phases[phase].appHost,
-        DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
+        // Node
         NODE_ENV: phases[phase].env || 'dev',
+        // Elastic Search
         ELASTICSEARCH_URL: phases[phase].elasticsearchURL,
         ELASTICSEARCH_EML_INDEX: phases[phase].elasticsearchEmlIndex,
         ELASTICSEARCH_TAXONOMY_INDEX: phases[phase].elasticsearchTaxonomyIndex,
+        // S3 (Object Store)
         S3_KEY_PREFIX: phases[phase].s3KeyPrefix,
-        TZ: phases[phase].tz,
-        KEYCLOAK_ADMIN_USERNAME: phases[phase].sso.adminUserName,
-        KEYCLOAK_SECRET: phases[phase].sso.keycloakSecret,
-        KEYCLOAK_SECRET_ADMIN_PASSWORD: phases[phase].sso.keycloakSecretAdminPassword,
-        KEYCLOAK_HOST: phases[phase].sso.url,
-        KEYCLOAK_CLIENT_ID: phases[phase].sso.clientId,
-        KEYCLOAK_REALM: phases[phase].sso.realm,
-        KEYCLOAK_INTEGRATION_ID: phases[phase].sso.integrationId,
-        KEYCLOAK_API_HOST: phases[phase].sso.apiHost,
         OBJECT_STORE_SECRETS: 'biohubbc-object-store',
+        // Database
+        TZ: phases[phase].tz,
+        DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
+        // Log Level
         LOG_LEVEL: phases[phase].logLevel || 'info',
+        // Openshift Resources
         CPU_REQUEST: phases[phase].cpuRequest,
         CPU_LIMIT: phases[phase].cpuLimit,
         MEMORY_REQUEST: phases[phase].memoryRequest,