SIMSBIOHUB-584 Bulk Animals

api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts

@@ -0,0 +1,159 @@
+import { RequestHandler } from 'express';
+import { Operation } from 'express-openapi';
+import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
+import { getDBConnection } from '../../../../../../database/db';
+import { HTTP400 } from '../../../../../../errors/http-error';
+import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
+import { ImportCrittersService } from '../../../../../../services/import-services/import-critters-service';
+import { scanFileForVirus } from '../../../../../../utils/file-utils';
+import { getLogger } from '../../../../../../utils/logger';
+import { parseMulterFile } from '../../../../../../utils/media/media-utils';
+
+const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/critters/import');
+
+export const POST: Operation = [
+  authorizeRequestHandler((req) => {
+    return {
+      or: [
+        {
+          validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR],
+          surveyId: Number(req.params.surveyId),
+          discriminator: 'ProjectPermission'
+        },
+        {
+          validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR],
+          discriminator: 'SystemRole'
+        }
+      ]
+    };
+  }),
+  importCsv()
+];
+
+POST.apiDoc = {
+  description: 'Upload survey critters submission file',
+  tags: ['observations'],
+  security: [
+    {
+      Bearer: []
+    }
+  ],
+  parameters: [
+    {
+      in: 'path',
+      name: 'projectId',
+      required: true
+    },
+    {
+      in: 'path',
+      name: 'surveyId',
+      required: true
+    }
+  ],
+  requestBody: {
+    description: 'Survey critters submission file to import',
+    content: {
+      'multipart/form-data': {
+        schema: {
+          type: 'object',
+          additionalProperties: false,
+          required: ['media'],
+          properties: {
+            media: {
+              description: 'Survey Critters submission import file.',
+              type: 'string',
+              format: 'binary'
+            }
+          }
+        }
+      }
+    }
+  },
+  responses: {
+    200: {
+      description: 'Import OK',
+      content: {
+        'application/json': {
+          schema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+              survey_critter_ids: {
+                type: 'array',
+                items: {
+                  type: 'number'
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    400: {
+      $ref: '#/components/responses/400'
+    },
+    401: {
+      $ref: '#/components/responses/401'
+    },
+    403: {
+      $ref: '#/components/responses/403'
+    },
+    500: {
+      $ref: '#/components/responses/500'
+    },
+    default: {
+      $ref: '#/components/responses/default'
+    }
+  }
+};
+
+/**
+ * Imports a `Critter CSV` which adds critters to `survey_critter` table and creates critters in Critterbase.
+ *
+ * @return {*}  {RequestHandler}
+ */
+export function importCsv(): RequestHandler {
+  return async (req, res) => {
+    const surveyId = Number(req.params.surveyId);
+    const rawFiles = req.files as Express.Multer.File[];
+    const rawFile = rawFiles[0];
+
+    const connection = getDBConnection(req['keycloak_token']);
+
+    try {
+      await connection.open();
+
+      if (rawFiles.length !== 1) {
+        throw new HTTP400('Invalid number of files included. Expected 1 CSV file.');
+      }
+
+      if (!rawFile?.originalname.endsWith('.csv')) {
+        throw new HTTP400('Invalid file type. Expected a CSV file.');
+      }
+
+      // Check for viruses / malware
+      const virusScanResult = await scanFileForVirus(rawFile);
+
+      if (!virusScanResult) {
+        throw new HTTP400('Malicious content detected, import cancelled.');
+      }
+
+      const csvImporter = new ImportCrittersService(connection);
+
+      // Pass the survey id and the csv (MediaFile) to the importer
+      const surveyCritterIds = await csvImporter.import(surveyId, parseMulterFile(rawFile));
+
+      defaultLog.info({ label: 'importCritterCsv', message: 'result', survey_critter_ids: surveyCritterIds });
+
+      await connection.commit();
+
+      return res.status(200).json({ survey_critter_ids: surveyCritterIds });
+    } catch (error) {
+      defaultLog.error({ label: 'uploadMedia', message: 'error', error });
+      await connection.rollback();
+      throw error;
+    } finally {
+      connection.release();
+    }
+  };
+}

api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.test.ts

@@ -2,7 +2,7 @@ import { expect } from 'chai';
 import sinon from 'sinon';
 import { addCritterToSurvey, getCrittersFromSurvey } from '.';
 import * as db from '../../../../../../database/db';
-import { CritterbaseService } from '../../../../../../services/critterbase-service';
+import { CritterbaseService, ICritter } from '../../../../../../services/critterbase-service';
 import { SurveyCritterService } from '../../../../../../services/survey-critter-service';
 import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';
 
@@ -23,7 +23,7 @@ describe('getCrittersFromSurvey', () => {
       .resolves([mockSurveyCritter]);
     const mockGetMultipleCrittersByIds = sinon
       .stub(CritterbaseService.prototype, 'getMultipleCrittersByIds')
-      .resolves([mockCBCritter]);
+      .resolves([mockCBCritter] as unknown as ICritter[]);
 
     const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
 

api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.ts

@@ -211,10 +211,12 @@ export function getCrittersFromSurvey(): RequestHandler {
     const surveyId = Number(req.params.surveyId);
     const connection = getDBConnection(req['keycloak_token']);
 
-    const surveyService = new SurveyCritterService(connection);
-    const critterbaseService = new CritterbaseService(user);
     try {
       await connection.open();
+
+      const surveyService = new SurveyCritterService(connection);
+      const critterbaseService = new CritterbaseService(user);
+
       const surveyCritters = await surveyService.getCrittersInSurvey(surveyId);
 
       // Exit early if surveyCritters list is empty

api/src/repositories/survey-critter-repository.test.ts

@@ -26,16 +26,16 @@ describe('SurveyRepository', () => {
     });
   });
 
-  describe('addCritterToSurvey', () => {
+  describe('addCrittersToSurvey', () => {
     it('should return result', async () => {
-      const mockResponse = { rows: [{ submissionId: 1 }], rowCount: 1 } as any as Promise<QueryResult<any>>;
+      const mockResponse = { rows: [{ critter_id: 1 }], rowCount: 1 } as any as Promise<QueryResult<any>>;
       const dbConnection = getMockDBConnection({ knex: () => mockResponse });
 
       const repository = new SurveyCritterRepository(dbConnection);
 
-      const response = await repository.addCritterToSurvey(1, 'critter_id');
+      const response = await repository.addCrittersToSurvey(1, ['critter_id']);
 
-      expect(response).to.be.undefined;
+      expect(response).to.be.deep.equal([1]);
     });
   });
 

api/src/repositories/survey-critter-repository.ts

@@ -39,24 +39,24 @@ export class SurveyCritterRepository extends BaseRepository {
   }
 
   /**
-   * Add critter to survey
+   * Add critters to survey.
    *
    * @param {number} surveyId
-   * @param {string} critterId
+   * @param {string[]} critterIds
    * @return {*}  {Promise<number>}
    * @memberof SurveyCritterRepository
    */
-  async addCritterToSurvey(surveyId: number, critterId: string): Promise<number> {
-    defaultLog.debug({ label: 'addCritterToSurvey', surveyId });
+  async addCrittersToSurvey(surveyId: number, critterIds: string[]): Promise<number[]> {
+    defaultLog.debug({ label: 'addCrittersToSurvey', surveyId });
 
     const queryBuilder = getKnex()
       .table('critter')
-      .insert({ survey_id: surveyId, critterbase_critter_id: critterId })
+      .insert(critterIds.map((critterId) => ({ survey_id: surveyId, critterbase_critter_id: critterId })))
       .returning('critter_id');
 
     const response = await this.connection.knex(queryBuilder);
 
-    return response.rows[0].critter_id;
+    return response.rows.map((row) => row.critter_id);
   }
 
   /**

api/src/services/critterbase-service.test.ts

@@ -2,7 +2,7 @@ import { AxiosResponse } from 'axios';
 import chai, { expect } from 'chai';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
-import { CritterbaseService, ICritter } from './critterbase-service';
+import { CritterbaseService, ICreateCritter } from './critterbase-service';
 import { KeycloakService } from './keycloak-service';
 
 chai.use(sinonChai);
@@ -145,12 +145,11 @@ describe('CritterbaseService', () => {
 
     describe('createCritter', () => {
       it('should create a critter', async () => {
-        const data: ICritter = {
+        const data: ICreateCritter = {
           wlh_id: 'aaaa',
           animal_id: 'aaaa',
           sex: 'male',
           itis_tsn: 1,
-          itis_scientific_name: 'Name',
           critter_comment: 'None.'
         };
         const axiosStub = sinon.stub(cb.axiosInstance, 'post').resolves({ data: [] });

api/src/services/critterbase-service.ts

@@ -16,13 +16,22 @@ export interface QueryParam {
 }
 
 export interface ICritter {
-  critter_id?: string;
-  wlh_id: string;
-  animal_id: string;
+  critter_id: string;
+  wlh_id: string | null;
+  animal_id: string | null;
   sex: string;
   itis_tsn: number;
   itis_scientific_name: string;
-  critter_comment: string;
+  critter_comment: string | null;
+}
+
+export interface ICreateCritter {
+  critter_id?: string;
+  wlh_id?: string | null;
+  animal_id: string; // NOTE: In critterbase this is optional. For SIMS it should be required.
+  sex: string;
+  itis_tsn: number;
+  critter_comment?: string | null;
 }
 
 export interface ICapture {
@@ -110,15 +119,41 @@ export interface ICollection {
 }
 
 export interface IBulkCreate {
-  critters: ICritter[];
-  captures: ICapture[];
-  collections: ICollection[];
-  mortalities: IMortality[];
-  locations: ILocation[];
-  markings: IMarking[];
-  quantitative_measurements: IQuantMeasurement[];
-  qualitative_measurements: IQualMeasurement[];
-  families: IFamilyPayload[];
+  critters?: ICreateCritter[];
+  captures?: ICapture[];
+  collections?: ICollection[];
+  mortalities?: IMortality[];
+  locations?: ILocation[];
+  markings?: IMarking[];
+  quantitative_measurements?: IQuantMeasurement[];
+  qualitative_measurements?: IQualMeasurement[];
+  families?: IFamilyPayload[];
+}
+
+interface IBulkResponse {
+  critters: number;
+  captures: number;
+  collections: number;
+  mortalities: number;
+  locations: number;
+  markings: number;
+  quantitative_measurements: number;
+  qualitative_measurements: number;
+  families: number;
+  family_parents: number;
+  family_chidren: number;
+}
+
+export interface IBulkCreateResponse {
+  created: IBulkResponse;
+}
+
+export interface ICollectionUnitWithCategory {
+  collection_unit_id: string;
+  collection_category_id: string;
+  category_name: string;
+  unit_name: string;
+  description: string | null;
 }
 
 /**
@@ -366,7 +401,7 @@ export class CritterbaseService {
     return this._makeGetRequest(`${CRITTER_ENDPOINT}/${critter_id}`, [{ key: 'format', value: 'detail' }]);
   }
 
-  async createCritter(data: ICritter) {
+  async createCritter(data: ICreateCritter) {
     const response = await this.axiosInstance.post(`${CRITTER_ENDPOINT}/create`, data);
     return response.data;
   }
@@ -376,7 +411,12 @@ export class CritterbaseService {
     return response.data;
   }
 
-  async getMultipleCrittersByIds(critter_ids: string[]) {
+  async bulkCreate(data: IBulkCreate): Promise<IBulkCreateResponse> {
+    const response = await this.axiosInstance.post(BULK_ENDPOINT, data);
+    return response.data;
+  }
+
+  async getMultipleCrittersByIds(critter_ids: string[]): Promise<ICritter[]> {
     const response = await this.axiosInstance.post(CRITTER_ENDPOINT, { critter_ids });
     return response.data;
   }
@@ -419,4 +459,30 @@ export class CritterbaseService {
 
     return data;
   }
+
+  /**
+   * Find collection categories by tsn. Includes hierarchies.
+   *
+   * @async
+   * @param {string} tsn - ITIS TSN
+   * @returns {Promise<>} Collection categories
+   */
+  async findTaxonCollectionCategories(tsn: string) {
+    const response = await this.axiosInstance.get(`/xref/taxon-collection-categories?tsn=${tsn}`);
+
+    return response.data;
+  }
+
+  /**
+   * Find collection units by tsn. Includes hierarchies.
+   *
+   * @async
+   * @param {string} tsn - ITIS TSN
+   * @returns {Promise<ICollectionUnitWithCategory[]>} Collection units
+   */
+  async findTaxonCollectionUnits(tsn: string): Promise<ICollectionUnitWithCategory[]> {
+    const response = await this.axiosInstance.get(`/xref/taxon-collection-units?tsn=${tsn}`);
+
+    return response.data;
+  }
 }