SIMSBIOHUB-374: BioHub Feature Submission Schema (#209)

* Submission Feature Migrations and Test Seeding
Remove DB_SCHEMA usage in most places.
Remove DB Views
Add Mock test data seed (disabled by default)

.github/workflows/cleanClosedPR.yml

@@ -86,5 +86,5 @@ jobs:
         env:
           POD_SELECTOR: biohub-platform
         run: |
-          oc --namespace a0ec71-dev   get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash
-          oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash
+          oc --namespace a0ec71-dev   get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash
+          oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash

.github/workflows/deploy.yml

@@ -5,6 +5,8 @@ name: PR-Based Deploy on OpenShift
 on:
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
+    branches-ignore:
+      - prod
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.number }}
@@ -41,9 +43,11 @@ jobs:
       - checkEnv
     outputs:
       paths_result: ${{ steps.skip_check.outputs.paths_result }}
-      # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message.
+      # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message OR the base branch
+      # is dev, test, or prod.
       # Used to disable duplicate action skipping, if needed.
-      ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') }}
+      ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') || 
+        github.head_ref == 'dev' || github.head_ref == 'test' || github.head_ref == 'prod' }}
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@v5

Makefile

@@ -56,6 +56,14 @@ clean: ## Closes and cleans (removes) all project containers
 	@echo "==============================================="
 	@docker-compose -f docker-compose.yml down -v --rmi all --remove-orphans
 
+prune: ## Deletes ALL docker artifacts (even those not associated to this project)
+	@echo -n "Delete ALL docker artifacts? [y/n] " && read ans && [ $${ans:-n} = y ]
+	@echo "==============================================="
+	@echo "Make: prune - deleting all docker artifacts"
+	@echo "==============================================="
+	@docker system prune --all --volumes -f
+	@docker volume prune --all -f
+
 ## ------------------------------------------------------------------------------
 ## Build/Run Postgres DB Commands
 ## - Builds all of the BioHub postgres db projects (db, db_setup)
@@ -82,13 +90,15 @@ build-backend: ## Builds all backend containers
 	@echo "==============================================="
 	@echo "Make: build-backend - building backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api queue
+	@docker-compose -f docker-compose.yml build db db_setup api
+	# @docker-compose -f docker-compose.yml build db db_setup api queue
 
 run-backend: ## Runs all backend containers
 	@echo "==============================================="
 	@echo "Make: run-backend - running backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api queue
+	@docker-compose -f docker-compose.yml up -d db db_setup api
+	# @docker-compose -f docker-compose.yml up -d db db_setup api queue
 
 ## ------------------------------------------------------------------------------
 ## Build/Run Backend+Web Commands (backend + web frontend)
@@ -99,13 +109,15 @@ build-web: ## Builds all backend+web containers
 	@echo "==============================================="
 	@echo "Make: build-web - building web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api queue app
+	@docker-compose -f docker-compose.yml build db db_setup api app
+	# @docker-compose -f docker-compose.yml build db db_setup api queue app
 
 run-web: ## Runs all backend+web containers
 	@echo "==============================================="
 	@echo "Make: run-web - running web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api queue app
+	@docker-compose -f docker-compose.yml up -d db db_setup api app
+	# @docker-compose -f docker-compose.yml up -d db db_setup api queue app
 
 ## ------------------------------------------------------------------------------
 ## Commands to shell into the target container

api/.pipeline/config.js

@@ -62,9 +62,9 @@ const phases = {
     env: 'build',
     tz: config.timezone.api,
     branch: branch,
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '1250m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '3Gi'
   },
   dev: {
@@ -87,9 +87,10 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.dev,
     logLevel: 'debug',
-    cpuRequest: '100m',
+    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '500m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '1',
     replicasMax: (isStaticDeployment && '2') || '1'
@@ -114,9 +115,10 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.test,
     logLevel: 'info',
-    cpuRequest: '200m',
+    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '1000m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '2',
     replicasMax: '3'
@@ -141,9 +143,10 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.prod,
     logLevel: 'info',
-    cpuRequest: '200m',
+    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '1000m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '2',
     replicasMax: '3'

api/.pipeline/lib/api.deploy.js

@@ -31,22 +31,30 @@ const apiDeploy = async (settings) => {
         HOST: phases[phase].host,
         CHANGE_ID: phases.build.changeId || changeId,
         APP_HOST: phases[phase].appHost,
+        // Node
         NODE_ENV: phases[phase].env || 'dev',
+        NODE_OPTIONS: phases[phase].nodeOptions,
+        // Elastic Search
         ELASTICSEARCH_URL: phases[phase].elasticsearchURL,
         ELASTICSEARCH_EML_INDEX: phases[phase].elasticsearchEmlIndex,
         ELASTICSEARCH_TAXONOMY_INDEX: phases[phase].elasticsearchTaxonomyIndex,
+        // S3 (Object Store)
         S3_KEY_PREFIX: phases[phase].s3KeyPrefix,
+        // Database
         TZ: phases[phase].tz,
+        DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
+        // Keycloak
         KEYCLOAK_ADMIN_USERNAME: phases[phase].sso.adminUserName,
         KEYCLOAK_SECRET: phases[phase].sso.keycloakSecret,
         KEYCLOAK_SECRET_ADMIN_PASSWORD: phases[phase].sso.keycloakSecretAdminPassword,
-        DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
         KEYCLOAK_HOST: phases[phase].sso.url,
         KEYCLOAK_CLIENT_ID: phases[phase].sso.clientId,
         KEYCLOAK_REALM: phases[phase].sso.realm,
         KEYCLOAK_INTEGRATION_ID: phases[phase].sso.integrationId,
         KEYCLOAK_API_HOST: phases[phase].sso.apiHost,
+        // Log Level
         LOG_LEVEL: phases[phase].logLevel || 'info',
+        // OPenshift Resources
         CPU_REQUEST: phases[phase].cpuRequest,
         CPU_LIMIT: phases[phase].cpuLimit,
         MEMORY_REQUEST: phases[phase].memoryRequest,

api/.pipeline/templates/api.dc.yaml

@@ -35,6 +35,7 @@ parameters:
     description: Application Environment type variable
     required: true
     value: 'dev'
+  - name: NODE_OPTIONS
   - name: ELASTICSEARCH_URL
     description: Platform Elasticsearch URL
     required: true
@@ -223,6 +224,8 @@ objects:
                   value: ${CHANGE_ID}
                 - name: NODE_ENV
                   value: ${NODE_ENV}
+                - name: NODE_OPTIONS
+                  value: ${NODE_OPTIONS}
                 - name: ELASTICSEARCH_URL
                   value: ${ELASTICSEARCH_URL}
                 - name: ELASTICSEARCH_EML_INDEX