SIMSBIOHUB-459: Fixed save taxonomy repo method (#236)

* Fixed a race condition that occurs when two requests try to read/write from the taxon table.
bcgov · Feb 12, 2024 · 2f5cd0d · 2f5cd0d
1 parent e761319
commit 2f5cd0d
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 33 deletions.
diff --git a/api/.pipeline/config.js b/api/.pipeline/config.js
@@ -87,13 +87,13 @@ const phases = {
     s3KeyPrefix: (isStaticDeployment && 'biohub') || `local/${deployChangeId}/biohub`,
     tz: config.timezone.api,
     sso: config.sso.dev,
-    logLevel: 'debug',
-    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    logLevel: (isStaticDeployment && 'info') || 'debug',
+    nodeOptions: '--max_old_space_size=2250', // 75% of memoryLimit (bytes)
     cpuRequest: '50m',
-    cpuLimit: '500m',
+    cpuLimit: '600m',
     memoryRequest: '100Mi',
-    memoryLimit: '2Gi',
-    replicas: '1',
+    memoryLimit: '3Gi',
+    replicas: (isStaticDeployment && '1') || '1',
     replicasMax: (isStaticDeployment && '2') || '1'
   },
   test: {
@@ -117,13 +117,13 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.test,
     logLevel: 'info',
-    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    nodeOptions: '--max_old_space_size=2250', // 75% of memoryLimit (bytes)
     cpuRequest: '50m',
     cpuLimit: '1000m',
     memoryRequest: '100Mi',
-    memoryLimit: '2Gi',
+    memoryLimit: '3Gi',
     replicas: '2',
-    replicasMax: '3'
+    replicasMax: '4'
   },
   prod: {
     namespace: 'a0ec71-prod',
@@ -145,14 +145,14 @@ const phases = {
     s3KeyPrefix: 'biohub',
     tz: config.timezone.api,
     sso: config.sso.prod,
-    logLevel: 'info',
-    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    logLevel: 'warn',
+    nodeOptions: '--max_old_space_size=2250', // 75% of memoryLimit (bytes)
     cpuRequest: '50m',
     cpuLimit: '1000m',
     memoryRequest: '100Mi',
-    memoryLimit: '2Gi',
+    memoryLimit: '3Gi',
     replicas: '2',
-    replicasMax: '3'
+    replicasMax: '4'
   }
 };
 

diff --git a/api/src/app.ts b/api/src/app.ts
@@ -28,7 +28,7 @@ const app: express.Express = express();
 
 // Enable CORS
 app.use(function (req: Request, res: Response, next: NextFunction) {
-  defaultLog.info(`${req.method} ${req.url}`);
+  defaultLog.debug(`${req.method} ${req.url}`);
 
   res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With, Content-Type, Authorization, responseType');
   res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE, HEAD');

diff --git a/api/src/repositories/taxonomy-repository.ts b/api/src/repositories/taxonomy-repository.ts
@@ -2,8 +2,11 @@ import SQL from 'sql-template-strings';
 import { z } from 'zod';
 import { getKnex } from '../database/db';
 import { ApiExecuteSQLError } from '../errors/api-error';
+import { getLogger } from '../utils/logger';
 import { BaseRepository } from './base-repository';
 
+const defaultLog = getLogger('repositories/taxonomy-repository');
+
 export const TaxonRecord = z.object({
   taxon_id: z.number(),
   itis_tsn: z.number(),
@@ -60,28 +63,40 @@ export class TaxonomyRepository extends BaseRepository {
     itisTsn: number,
     itisScientificName: string,
     commonName: string | null,
-    itisData: Record<any, any>,
+    itisData: Record<string, unknown>,
     itisUpdateDate: string
   ): Promise<TaxonRecord> {
+    defaultLog.debug({ label: 'addItisTaxonRecord', itisTsn });
+
     const sqlStatement = SQL`
-      INSERT INTO
-        taxon
-      (
-        itis_tsn,
-        itis_scientific_name,
-        common_name,
-        itis_data,
-        itis_update_date
+      WITH inserted_row AS (
+        INSERT INTO
+          taxon 
+        (
+          itis_tsn,
+          itis_scientific_name,
+          common_name,
+          itis_data,
+          itis_update_date
+        )
+        VALUES (
+          ${itisTsn},
+          ${itisScientificName},
+          ${commonName},
+          ${itisData},
+          ${itisUpdateDate}
+        )
+        ON CONFLICT
+        DO NOTHING
+        RETURNING *
       )
-      VALUES (
-        ${itisTsn},
-        ${itisScientificName},
-        ${commonName},
-        ${itisData},
-        ${itisUpdateDate}
-      )
-      RETURNING
-        *;
+      SELECT * FROM inserted_row
+      UNION
+      SELECT * FROM taxon
+      WHERE 
+        taxon.itis_tsn = ${itisTsn}
+      AND
+        taxon.record_end_date IS null;
     `;
 
     const response = await this.connection.sql(sqlStatement, TaxonRecord);

diff --git a/api/src/services/taxonomy-service.ts b/api/src/services/taxonomy-service.ts
@@ -1,7 +1,10 @@
 import { IDBConnection } from '../database/db';
 import { TaxonomyRepository, TaxonRecord } from '../repositories/taxonomy-repository';
+import { getLogger } from '../utils/logger';
 import { ItisService, ItisSolrSearchResponse } from './itis-service';
 
+const defaultLog = getLogger('services/taxonomy-service');
+
 export type TaxonSearchResult = {
   tsn: number;
   commonName: string | null;
@@ -29,11 +32,15 @@ export class TaxonomyService {
    * @memberof TaxonomyService
    */
   async getTaxonByTsnIds(tsnIds: number[]): Promise<TaxonSearchResult[]> {
+    defaultLog.debug({ label: 'getTaxonByTsnIds', tsnIds });
+
     // Search for taxon records in the database
     const existingTaxonRecords = await this.taxonRepository.getTaxonByTsnIds(tsnIds);
-    let patchedTaxonRecords: TaxonRecord[] = [];
+    const existingTsnIds = existingTaxonRecords.map((record) => record.itis_tsn);
 
-    const missingTsnIds = tsnIds.filter((tsnId) => !existingTaxonRecords.find((item) => item.itis_tsn === tsnId));
+    const missingTsnIds = tsnIds.filter((tsnId) => !existingTsnIds.includes(tsnId));
+
+    let patchedTaxonRecords: TaxonRecord[] = [];
 
     if (missingTsnIds.length) {
       // If the local database does not contain a record for all of the requested ids, search ITIS for the missing