Merge pull request #1360 from bcgov/dev

Release dev to test

.config/config.json

@@ -5,6 +5,7 @@
     "tools": "tools",
     "dev": "dev",
     "test": "test",
+    "test-spi": "test",
     "prod": "prod"
   },
   "version": "1.0.0",
@@ -20,17 +21,20 @@
   "staticUrls": {
     "dev": "dev-biohubbc.apps.silver.devops.gov.bc.ca",
     "test": "test-biohubbc.apps.silver.devops.gov.bc.ca",
+    "test-spi": "test-spi-biohubbc.apps.silver.devops.gov.bc.ca",
     "prod": "biohubbc.apps.silver.devops.gov.bc.ca",
     "prodVanityUrl": "sims.nrs.gov.bc.ca"
   },
   "staticUrlsAPI": {
     "dev": "api-dev-biohubbc.apps.silver.devops.gov.bc.ca",
     "test": "api-test-biohubbc.apps.silver.devops.gov.bc.ca",
+    "test-spi": "api-test-spi-biohubbc.apps.silver.devops.gov.bc.ca",
     "prod": "api-biohubbc.apps.silver.devops.gov.bc.ca"
   },
   "siteminderLogoutURL": {
     "dev": "https://logontest7.gov.bc.ca/clp-cgi/logoff.cgi",
     "test": "https://logontest7.gov.bc.ca/clp-cgi/logoff.cgi",
+    "test-spi": "https://logontest7.gov.bc.ca/clp-cgi/logoff.cgi",
     "prod": "https://logon7.gov.bc.ca/clp-cgi/logoff.cgi"
   },
   "sso": {
@@ -68,6 +72,23 @@
         "cssApiEnvironment": "test"
       }
     },
+    "test-spi": {
+      "host": "https://test.loginproxy.gov.bc.ca/auth",
+      "realm": "standard",
+      "clientId": "sims-4461",
+      "keycloakSecret": "keycloak",
+      "serviceClient": {
+        "serviceClientName": "sims-svc-4464",
+        "keycloakSecretServiceClientPasswordKey": "sims_svc_client_password"
+      },
+      "cssApi": {
+        "cssApiTokenUrl": "https://loginproxy.gov.bc.ca/auth/realms/standard/protocol/openid-connect/token",
+        "cssApiClientId": "service-account-team-1190-4229",
+        "cssApiHost": "https://api.loginproxy.gov.bc.ca/api/v1",
+        "keycloakSecretCssApiSecretKey": "css_api_client_secret",
+        "cssApiEnvironment": "test"
+      }
+    },
     "prod": {
       "host": "https://loginproxy.gov.bc.ca/auth",
       "realm": "standard",

.github/workflows/deploy.yml

@@ -7,6 +7,7 @@ on:
     types: [opened, reopened, synchronize, ready_for_review]
     branches-ignore:
       - test
+      - test-spi
       - prod
 
 concurrency:

.github/workflows/deployStatic.yml

@@ -8,6 +8,7 @@ on:
     branches:
       - dev
       - test
+      - test-spi
       - prod
 
 jobs:

Makefile

@@ -6,7 +6,7 @@
 
 -include .env
 
-# Apply the contents of the .env to the terminal, so that the docker-compose file can use them in its builds
+# Apply the contents of the .env to the terminal, so that the compose file can use them in its builds
 export $(shell sed 's/=.*//' .env)
 
 ## ------------------------------------------------------------------------------
@@ -46,13 +46,13 @@ close: ## Closes all project containers
 	@echo "==============================================="
 	@echo "Make: close - closing Docker containers"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml down
+	@docker compose down
 
 clean: ## Closes and cleans (removes) all project containers
 	@echo "==============================================="
 	@echo "Make: clean - closing and cleaning Docker containers"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml down -v --rmi all --remove-orphans
+	@docker compose down -v --rmi all --remove-orphans
 
 prune: ## Deletes ALL docker artifacts (even those not associated to this project)
 	@echo -n "Delete ALL docker artifacts? [y/n] " && read ans && [ $${ans:-n} = y ]
@@ -77,13 +77,13 @@ build-postgres: ## Builds the postgres db containers
 	@echo "==============================================="
 	@echo "Make: build-postgres - building postgres db  images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup
+	@docker compose build db db_setup
 
 run-postgres: ## Runs the postgres db containers
 	@echo "==============================================="
 	@echo "Make: run-postgres - running postgres db  images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup
+	@docker compose up -d db db_setup
 
 ## ------------------------------------------------------------------------------
 ## Build/Run Backend Commands
@@ -94,13 +94,13 @@ build-backend: ## Builds all backend containers
 	@echo "==============================================="
 	@echo "Make: build-backend - building backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api
+	@docker compose build db db_setup api
 
 run-backend: ## Runs all backend containers
 	@echo "==============================================="
 	@echo "Make: run-backend - running backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api
+	@docker compose up -d db db_setup api
 
 ## ------------------------------------------------------------------------------
 ## Build/Run Backend+Web Commands (backend + web frontend)
@@ -111,13 +111,13 @@ build-web: ## Builds all backend+web containers
 	@echo "==============================================="
 	@echo "Make: build-web - building web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api app
+	@docker compose build db db_setup api app
 
 run-web: ## Runs all backend+web containers
 	@echo "==============================================="
 	@echo "Make: run-web - running web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api app
+	@docker compose up -d db db_setup api app
 
 ## ------------------------------------------------------------------------------
 ## Commands to shell into the target container
@@ -128,19 +128,19 @@ db-container: ## Executes into database container.
 	@echo "Make: Shelling into database container"
 	@echo "==============================================="
 	@export PGPASSWORD=$(DB_ADMIN_PASS)
-	@docker-compose exec db psql -U $(DB_ADMIN) -d $(DB_DATABASE)
+	@docker compose exec db psql -U $(DB_ADMIN) -d $(DB_DATABASE)
 
 app-container: ## Executes into the app container.
 	@echo "==============================================="
 	@echo "Shelling into app container"
 	@echo "==============================================="
-	@docker-compose exec app bash
+	@docker compose exec app bash
 
 api-container: ## Executes into the api container.
 	@echo "==============================================="
 	@echo "Shelling into api container"
 	@echo "==============================================="
-	@docker-compose exec api bash
+	@docker compose exec api bash
 
 ## ------------------------------------------------------------------------------
 ## Database migration commands
@@ -150,37 +150,37 @@ build-db-setup: ## Build the db knex setup (migrations + seeding) image
 	@echo "==============================================="
 	@echo "Make: build-db-setup - building db knex setup image"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db_setup
+	@docker compose build db_setup
 
 run-db-setup: ## Run the database migrations and seeding
 	@echo "==============================================="
 	@echo "Make: run-db-setup - running database migrations and seeding"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up db_setup
+	@docker compose up db_setup
 
 build-db-migrate: ## Build the db knex migrations image
 	@echo "==============================================="
 	@echo "Make: build-db-migrate - building db knex migrate image"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db_migrate
+	@docker compose build db_migrate
 
 run-db-migrate: ## Run the database migrations
 	@echo "==============================================="
 	@echo "Make: run-db-migrate - running database migrations"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up db_migrate
+	@docker compose up db_migrate
 
 build-db-rollback: ## Build the db knex rollback image
 	@echo "==============================================="
 	@echo "Make: build-db-rollback - building db knex rollback image"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db_rollback
+	@docker compose build db_rollback
 
 run-db-rollback: ## Rollback the latest database migrations
 	@echo "==============================================="
 	@echo "Make: run-db-rollback - rolling back the latest database migrations"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up db_rollback
+	@docker compose up db_rollback
 
 ## ------------------------------------------------------------------------------
 ## clamav commands
@@ -190,13 +190,13 @@ build-clamav: ## Build the clamav image
 	@echo "==============================================="
 	@echo "Make: build-clamav - building clamav image"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build clamav
+	@docker compose build clamav
 
 run-clamav: ## Run clamav
 	@echo "==============================================="
 	@echo "Make: run-clamav - running clamav"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d clamav
+	@docker compose up -d clamav
 
 ## ------------------------------------------------------------------------------
 ## Run `npm` commands for all projects
@@ -314,11 +314,11 @@ pipeline-install: ## Runs `npm install` for all projects
 
 args ?= --tail 2000 ## Default args if none are provided
 
-log: ## Runs `docker-compose logs -f` for all containers
+log: ## Runs `docker compose logs -f` for all containers
 	@echo "==============================================="
 	@echo "Running docker logs for the app container"
 	@echo "==============================================="
-	@docker-compose logs -f $(args)
+	@docker compose logs -f $(args)
 
 log-app: ## Runs `docker logs <container> -f` for the app container
 	@echo "==============================================="

README.md

@@ -125,7 +125,7 @@ Below are all of the relevant files that need to be updated when modifying envir
 #### Local Development
 
 - `env.docker`
-- `docker-compose.yml`
+- `compose.yml`
 - `app/src/contexts/configContext.tsx`
 
 #### Deployed to OpenShift

api/.docker/api/Dockerfile

@@ -1,5 +1,5 @@
 # ########################################################################################################
-# This DockerFile is used for local development (via docker-compose) only.
+# This DockerFile is used for local development (via compose.yml) only.
 # ########################################################################################################
 
 FROM node:20
@@ -21,6 +21,9 @@ ENV PATH ${HOME}/node_modules/.bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/us
 # Copy the rest of the files
 COPY . ./
 
+# Update log directory file permissions, prevents permission errors for linux environments
+RUN chmod -R a+rw data/logs/*
+
 VOLUME ${HOME}
 
 # start api with live reload

api/.gitignore

@@ -12,6 +12,9 @@ dist
 # Testing
 coverage
 
+# persistent storage
+data
+
 # SonarQube
 .sonarqube
 

api/.pipeline/config.js

@@ -52,7 +52,6 @@ const phases = {
   build: {
     namespace: 'af2668-tools',
     name: `${name}`,
-    dbName: `${dbName}`,
     phase: 'build',
     changeId: changeId,
     suffix: `-build-${changeId}`,
@@ -91,7 +90,14 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.dev,
     featureFlags: '',
-    logLevel: 'info',
+    logLevel: (isStaticDeployment && 'info') || 'debug',
+    logLevelFile: (isStaticDeployment && 'debug') || 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: (isStaticDeployment && '10') || '2',
+    volumeCapacity: (isStaticDeployment && '500Mi') || '100Mi',
     apiResponseValidationEnabled: true,
     databaseResponseValidationEnabled: true,
     nodeOptions: '--max_old_space_size=3000', // 75% of memoryLimit (bytes)
@@ -126,8 +132,58 @@ const phases = {
     s3KeyPrefix: 'sims',
     tz: config.timezone.api,
     sso: config.sso.test,
-    logLevel: 'info',
     featureFlags: '',
+    logLevel: 'warn',
+    logLevelFile: 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: '10',
+    volumeCapacity: '500Mi',
+    apiResponseValidationEnabled: true,
+    databaseResponseValidationEnabled: true,
+    nodeOptions: '--max_old_space_size=3000', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
+    cpuLimit: '1000m',
+    memoryRequest: '100Mi',
+    memoryLimit: '4Gi',
+    replicas: '2',
+    replicasMax: '2'
+  },
+  'test-spi': {
+    namespace: 'af2668-test',
+    name: `${name}-spi`,
+    dbName: `${dbName}-spi`,
+    phase: 'test-spi',
+    changeId: deployChangeId,
+    suffix: `-test-spi`,
+    instance: `${name}-spi-test-spi`,
+    version: `${version}`,
+    tag: `test-spi-${version}`,
+    host: staticUrlsAPI['test-spi'],
+    appHost: staticUrls['test-spi'],
+    backboneInternalApiHost: 'https://api-test-biohub-platform.apps.silver.devops.gov.bc.ca',
+    backbonePublicApiHost: 'https://api-test-biohub-platform.apps.silver.devops.gov.bc.ca',
+    backboneIntakePath: '/api/submission/intake',
+    backboneArtifactIntakePath: '/api/artifact/intake',
+    biohubTaxonPath: '/api/taxonomy/taxon',
+    biohubTaxonTsnPath: '/api/taxonomy/taxon/tsn',
+    bctwApiHost: 'https://moe-bctw-api-test.apps.silver.devops.gov.bc.ca',
+    critterbaseApiHost: 'https://moe-critterbase-api-test.apps.silver.devops.gov.bc.ca/api',
+    nodeEnv: 'production',
+    s3KeyPrefix: 'sims',
+    tz: config.timezone.api,
+    sso: config.sso['test-spi'],
+    featureFlags: '',
+    logLevel: 'warn',
+    logLevelFile: 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: '10',
+    volumeCapacity: '500Mi',
     apiResponseValidationEnabled: true,
     databaseResponseValidationEnabled: true,
     nodeOptions: '--max_old_space_size=3000', // 75% of memoryLimit (bytes)
@@ -163,7 +219,14 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.prod,
     featureFlags: 'API_FF_SUBMIT_BIOHUB',
-    logLevel: 'warn',
+    logLevel: 'silent',
+    logLevelFile: 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: '10',
+    volumeCapacity: '500Mi',
     apiResponseValidationEnabled: false,
     databaseResponseValidationEnabled: false,
     nodeOptions: '--max_old_space_size=6000', // 75% of memoryLimit (bytes)

api/.pipeline/lib/api.deploy.js

@@ -34,6 +34,8 @@ const apiDeploy = async (settings) => {
         // Node
         NODE_ENV: phases[phase].nodeEnv,
         NODE_OPTIONS: phases[phase].nodeOptions,
+        // Persistent Volume
+        VOLUME_CAPACITY: phases[phase].volumeCapacity,
         // BioHub Platform (aka: Backbone)
         BACKBONE_INTERNAL_API_HOST: phases[phase].backboneInternalApiHost,
         BACKBONE_INTAKE_PATH: phases[phase].backboneIntakePath,
@@ -65,6 +67,13 @@ const apiDeploy = async (settings) => {
         KEYCLOAK_API_ENVIRONMENT: phases[phase].sso.cssApi.cssApiEnvironment,
         // Log Level
         LOG_LEVEL: phases[phase].logLevel,
+        LOG_LEVEL_FILE: phases[phase].logLevelFile,
+        LOG_FILE_DIR: phases[phase].logFileDir,
+        LOG_FILE_NAME: phases[phase].logFileName,
+        LOG_FILE_DATE_PATTERN: phases[phase].logFileDatePattern,
+        LOG_FILE_MAX_SIZE: phases[phase].logFileMaxSize,
+        LOG_FILE_MAX_FILES: phases[phase].logFileMaxFiles,
+        // Api Validation
         API_RESPONSE_VALIDATION_ENABLED: phases[phase].apiResponseValidationEnabled,
         DATABASE_RESPONSE_VALIDATION_ENABLED: phases[phase].databaseResponseValidationEnabled,
         // Feature Flags