SPI ETL: Map SPI Species Codes to ITIS TSNs (#1362)

* add species map

* linter

* cleanup and comments

Makefile

@@ -30,8 +30,6 @@ db-migrate: | build-db-migrate run-db-migrate ## Performs all commands necessary
 db-rollback: | build-db-rollback run-db-rollback ## Performs all commands necessary to rollback the latest database migrations
 clamav: | build-clamav run-clamav ## Performs all commands necessary to run clamav
 
-spi-transform: | build-spi-transform run-spi-transform ## Performs all commands to transform migrated SPI data
-
 fix: | lint-fix format-fix ## Performs both lint-fix and format-fix commands
 
 ## ------------------------------------------------------------------------------
@@ -385,3 +383,8 @@ trace-api:
 ## ------------------------------------------------------------------------------
 help: ## Display this help screen.
 	@grep -h -E '^[0-9a-zA-Z_-]+:.*?##.*$$|^##.*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-20s\033[0m %s\n", $$1, $$2}' | awk 'BEGIN {FS = "## "}; {printf "\033[36m%-1s\033[0m %s\n", $$2, $$1}'
+
+## ------------------------------------------------------------------------------
+## SPI Transformation for ETL
+## ------------------------------------------------------------------------------
+spi-transform: | build-spi-transform run-spi-transform ## Performs all commands to transform migrated SPI data

compose.yml

@@ -212,25 +212,17 @@ services:
     environment:
       - NODE_ENV=${NODE_ENV}
       # Database
-      # - DB_HOST=${DB_HOST}
-      # - DB_ADMIN=${DB_ADMIN}
-      # - DB_ADMIN_PASS=${DB_ADMIN_PASS}
-      # - DB_PORT=5432
-      # - DB_DATABASE=${DB_DATABASE}
-      # - DB_SCHEMA=${DB_SCHEMA}
-      # - DB_USER_API=${DB_USER_API}
-      # - DB_USER_API_PASS=${DB_USER_API_PASS}
       - DB_HOST=${SPI_DB_HOST}
-      - DB_ADMIN=${SPI_DB_ADMIN}
+      - DB_ADMIN=${DB_ADMIN}
       - DB_ADMIN_PASS=${SPI_DB_ADMIN_PASS}
       - DB_PORT=${SPI_DB_PORT}
-      - DB_DATABASE=${SPI_DB_DATABASE}
-      - DB_SCHEMA=${SPI_DB_SCHEMA}
-      - DB_USER_API=${SPI_DB_USER_API}
-      - DB_USER_API_PASS=${SPI_DB_USER_API_PASS}
-      - DB_SCHEMA_DAPI_V1=${SPI_DB_SCHEMA_DAPI_V1}
+      - DB_DATABASE=${DB_DATABASE}
+      - DB_SCHEMA=${DB_SCHEMA}
+      - DB_USER_API=${DB_USER_API}
+      - DB_USER_API_PASS=${DB_USER_API_PASS}
       - DB_CONNECTION_TIMEOUT=${DB_CONNECTION_TIMEOUT}
       - DB_POOL_SIZE=${DB_POOL_SIZE}
+      - SPI_MIGRATE_INCLUDE_SPECIES=${SPI_MIGRATE_INCLUDE_SPECIES}
     volumes:
       - /opt/app-root/src/node_modules
     networks:

database/src/migrations/20240906142100_spi_species_table.ts

@@ -0,0 +1,36 @@
+import { Knex } from 'knex';
+
+/**
+ * Add columns to the following tables for the SPI data migration:
+ * - project
+ * - survey
+ * - study species
+ * @export
+ * @param {Knex} knex
+ * @return {*}  {Promise<void>}
+ */
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(`--sql
+    SET SEARCH_PATH=public;
+
+    ----------------------------------------------------------------------------------------
+    -- Table that the SPI ETL scripts insert into, for mapping SPI species codes to ITIS TSNs
+    ----------------------------------------------------------------------------------------
+    CREATE TABLE IF NOT EXISTS public.migrate_spi_species (
+        id                      integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1),
+        spi_species_id          INTEGER NOT NULL,
+        spi_species_code        VARCHAR(128),
+        spi_scientific_name     VARCHAR(128),
+        spi_rank                VARCHAR(24),
+        itis_tsn                VARCHAR(16),
+        itis_scientific_name    VARCHAR(128),
+        itis_rank               VARCHAR(24),
+        CONSTRAINT itis_tsn_uk UNIQUE (itis_tsn),
+        CONSTRAINT spi_species_id_uk UNIQUE (spi_species_id)
+    );
+    `);
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw(``);
+}

spi/.gitignore

@@ -23,4 +23,4 @@ coverage
 
 npm-debug.log*
 yarn-debug.log*
-yarn-error.log*
+yarn-error.log*