diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts new file mode 100644 index 0000000000..49a2b7f87e --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts @@ -0,0 +1,213 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as db from '../../../../../../database/db'; +import { HTTP400 } from '../../../../../../errors/http-error'; +import { ImportCrittersService } from '../../../../../../services/import-services/import-critters-service'; +import { parseMulterFile } from '../../../../../../utils/media/media-utils'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db'; +import { importCsv } from './import'; + +import * as fileUtils from '../../../../../../utils/file-utils'; + +describe('importCsv', () => { + afterEach(() => { + sinon.restore(); + }); + + it('returns imported critters', async () => { + const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() }); + const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]); + const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true); + + const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File; + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = [mockFile]; + mockReq.params.surveyId = '1'; + + const requestHandler = importCsv(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(mockDBConnection.open).to.have.been.calledOnce; + + expect(mockFileScan).to.have.been.calledOnceWithExactly(mockFile); + + expect(mockGetDBConnection.calledOnce).to.be.true; + expect(mockImportCsv).to.have.been.calledOnceWithExactly(1, parseMulterFile(mockFile)); + expect(mockRes.json).to.have.been.calledOnceWithExactly({ survey_critter_ids: [1, 2] }); + + expect(mockDBConnection.commit).to.have.been.calledOnce; + expect(mockDBConnection.release).to.have.been.calledOnce; + }); + + it('should catch error and rollback if no files in request', async () => { + const mockDBConnection = getMockDBConnection({ + open: sinon.stub(), + commit: sinon.stub(), + release: sinon.stub(), + rollback: sinon.stub() + }); + const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]); + const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = []; + mockReq.params.surveyId = '1'; + + const requestHandler = importCsv(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (err: any) { + expect(err.message).to.be.equal('Invalid number of files included. Expected 1 CSV file.'); + } + + expect(mockDBConnection.open).to.have.been.calledOnce; + + expect(mockFileScan).to.not.have.been.calledOnce; + + expect(mockGetDBConnection.calledOnce).to.be.true; + expect(mockImportCsv).to.not.have.been.called; + expect(mockRes.json).to.not.have.been.called; + + expect(mockDBConnection.rollback).to.have.been.called; + expect(mockDBConnection.commit).to.not.have.been.called; + expect(mockDBConnection.release).to.have.been.called; + }); + + it('should catch error and rollback if more than 1 file in request', async () => { + const mockDBConnection = getMockDBConnection({ + open: sinon.stub(), + commit: sinon.stub(), + release: sinon.stub(), + rollback: sinon.stub() + }); + const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]); + const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true); + + const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File; + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = [mockFile, mockFile]; + mockReq.params.surveyId = '1'; + + const requestHandler = importCsv(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (err: any) { + expect(err.message).to.be.equal('Invalid number of files included. Expected 1 CSV file.'); + } + + expect(mockDBConnection.open).to.have.been.calledOnce; + + expect(mockFileScan).to.not.have.been.calledOnce; + + expect(mockGetDBConnection.calledOnce).to.be.true; + expect(mockImportCsv).to.not.have.been.called; + expect(mockRes.json).to.not.have.been.called; + + expect(mockDBConnection.rollback).to.have.been.called; + expect(mockDBConnection.commit).to.not.have.been.called; + expect(mockDBConnection.release).to.have.been.called; + }); + + it('should catch error and rollback if file is not a csv', async () => { + const mockDBConnection = getMockDBConnection({ + open: sinon.stub(), + commit: sinon.stub(), + release: sinon.stub(), + rollback: sinon.stub() + }); + const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]); + const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true); + + const mockFile = { + originalname: 'test.oops', + mimetype: 'test.oops', + buffer: Buffer.alloc(1) + } as Express.Multer.File; + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = [mockFile]; + mockReq.params.surveyId = '1'; + + const requestHandler = importCsv(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (err: any) { + expect(err).to.be.instanceof(HTTP400); + expect(err.message).to.be.contains('Invalid file type.'); + } + + expect(mockDBConnection.open).to.have.been.calledOnce; + + expect(mockFileScan).to.not.have.been.calledOnce; + + expect(mockGetDBConnection.calledOnce).to.be.true; + expect(mockImportCsv).to.not.have.been.called; + expect(mockRes.json).to.not.have.been.called; + + expect(mockDBConnection.rollback).to.have.been.called; + expect(mockDBConnection.commit).to.not.have.been.called; + expect(mockDBConnection.release).to.have.been.called; + }); + + it('should catch error and rollback if file contains malware', async () => { + const mockDBConnection = getMockDBConnection({ + open: sinon.stub(), + commit: sinon.stub(), + release: sinon.stub(), + rollback: sinon.stub() + }); + const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]); + + const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(false); + + const mockFile = { + originalname: 'test.csv', + mimetype: 'test.csv', + buffer: Buffer.alloc(1) + } as Express.Multer.File; + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + mockReq.files = [mockFile]; + mockReq.params.surveyId = '1'; + + const requestHandler = importCsv(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (err: any) { + expect(err).to.be.instanceof(HTTP400); + expect(err.message).to.be.contains('Malicious content detected'); + } + + expect(mockDBConnection.open).to.have.been.calledOnce; + expect(mockFileScan).to.have.been.calledOnceWithExactly(mockFile); + + expect(mockGetDBConnection.calledOnce).to.be.true; + expect(mockImportCsv).to.not.have.been.called; + expect(mockRes.json).to.not.have.been.called; + + expect(mockDBConnection.rollback).to.have.been.called; + expect(mockDBConnection.commit).to.not.have.been.called; + expect(mockDBConnection.release).to.have.been.called; + }); +}); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts new file mode 100644 index 0000000000..ed1e690c52 --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts @@ -0,0 +1,169 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles'; +import { getDBConnection } from '../../../../../../database/db'; +import { HTTP400 } from '../../../../../../errors/http-error'; +import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization'; +import { ImportCrittersService } from '../../../../../../services/import-services/import-critters-service'; +import { scanFileForVirus } from '../../../../../../utils/file-utils'; +import { getLogger } from '../../../../../../utils/logger'; +import { parseMulterFile } from '../../../../../../utils/media/media-utils'; + +const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/critters/import'); + +export const POST: Operation = [ + authorizeRequestHandler((req) => { + return { + or: [ + { + validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR], + surveyId: Number(req.params.surveyId), + discriminator: 'ProjectPermission' + }, + { + validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + importCsv() +]; + +POST.apiDoc = { + description: 'Upload survey critters submission file', + tags: ['critterbase', 'survey'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + in: 'path', + name: 'projectId', + required: true, + schema: { + type: 'integer', + minimum: 1 + } + }, + { + in: 'path', + name: 'surveyId', + required: true, + schema: { + type: 'integer', + minimum: 1 + } + } + ], + requestBody: { + description: 'Survey critters submission file to import', + content: { + 'multipart/form-data': { + schema: { + type: 'object', + additionalProperties: false, + required: ['media'], + properties: { + media: { + description: 'Survey Critters submission import file.', + type: 'string', + format: 'binary' + } + } + } + } + } + }, + responses: { + 200: { + description: 'Import OK', + content: { + 'application/json': { + schema: { + type: 'object', + additionalProperties: false, + required: ['survey_critter_ids'], + properties: { + survey_critter_ids: { + type: 'array', + items: { + type: 'integer', + minimum: 1 + } + } + } + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 403: { + $ref: '#/components/responses/403' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Imports a `Critter CSV` which adds critters to `survey_critter` table and creates critters in Critterbase. + * + * @return {*} {RequestHandler} + */ +export function importCsv(): RequestHandler { + return async (req, res) => { + const surveyId = Number(req.params.surveyId); + const rawFiles = req.files as Express.Multer.File[]; + const rawFile = rawFiles[0]; + + const connection = getDBConnection(req['keycloak_token']); + + try { + await connection.open(); + + if (rawFiles.length !== 1) { + throw new HTTP400('Invalid number of files included. Expected 1 CSV file.'); + } + + if (!rawFile?.originalname.endsWith('.csv')) { + throw new HTTP400('Invalid file type. Expected a CSV file.'); + } + + // Check for viruses / malware + const virusScanResult = await scanFileForVirus(rawFile); + + if (!virusScanResult) { + throw new HTTP400('Malicious content detected, import cancelled.'); + } + + const csvImporter = new ImportCrittersService(connection); + + // Pass the survey id and the csv (MediaFile) to the importer + const surveyCritterIds = await csvImporter.import(surveyId, parseMulterFile(rawFile)); + + defaultLog.info({ label: 'importCritterCsv', message: 'result', survey_critter_ids: surveyCritterIds }); + + await connection.commit(); + + return res.status(200).json({ survey_critter_ids: surveyCritterIds }); + } catch (error) { + defaultLog.error({ label: 'uploadMedia', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.test.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.test.ts index 9ff7fafa33..28036c38b0 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.test.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import sinon from 'sinon'; import { addCritterToSurvey, getCrittersFromSurvey } from '.'; import * as db from '../../../../../../database/db'; -import { CritterbaseService } from '../../../../../../services/critterbase-service'; +import { CritterbaseService, ICritter } from '../../../../../../services/critterbase-service'; import { SurveyCritterService } from '../../../../../../services/survey-critter-service'; import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db'; @@ -31,7 +31,7 @@ describe('getCrittersFromSurvey', () => { .resolves([mockSurveyCritter]); const mockGetMultipleCrittersByIds = sinon .stub(CritterbaseService.prototype, 'getMultipleCrittersByIds') - .resolves([mockCBCritter]); + .resolves([mockCBCritter] as unknown as ICritter[]); const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.ts index 49b7b82c43..13246cee5c 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/index.ts @@ -211,10 +211,10 @@ export function getCrittersFromSurvey(): RequestHandler { const surveyId = Number(req.params.surveyId); const connection = getDBConnection(req['keycloak_token']); - const surveyService = new SurveyCritterService(connection); - const critterbaseService = new CritterbaseService(user); try { await connection.open(); + + const surveyService = new SurveyCritterService(connection); const surveyCritters = await surveyService.getCrittersInSurvey(surveyId); // Exit early if surveyCritters list is empty @@ -223,6 +223,8 @@ export function getCrittersFromSurvey(): RequestHandler { } const critterIds = surveyCritters.map((critter) => String(critter.critterbase_critter_id)); + + const critterbaseService = new CritterbaseService(user); const result = await critterbaseService.getMultipleCrittersByIds(critterIds); const critterMap = new Map(); @@ -257,19 +259,20 @@ export function addCritterToSurvey(): RequestHandler { let critterId = req.body.critter_id; const connection = getDBConnection(req['keycloak_token']); - const surveyService = new SurveyCritterService(connection); - const cb = new CritterbaseService(user); try { await connection.open(); + const critterbaseService = new CritterbaseService(user); + // If request does not include critter ID, create a new critter and use its critter ID let result = null; if (!critterId) { - result = await cb.createCritter(req.body); + result = await critterbaseService.createCritter(req.body); critterId = result.critter_id; } + const surveyService = new SurveyCritterService(connection); const response = await surveyService.addCritterToSurvey(surveyId, critterId); await connection.commit(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}.ts index 1065625d40..ffa3837792 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}.ts @@ -95,12 +95,12 @@ export function updateSurveyCritter(): RequestHandler { const connection = getDBConnection(req['keycloak_token']); try { - await connection.open(); - if (!critterbaseCritterId) { throw new HTTPError(HTTPErrorType.BAD_REQUEST, 400, 'No external critter ID was found.'); } + await connection.open(); + const surveyService = new SurveyCritterService(connection); await surveyService.updateCritter(critterId, critterbaseCritterId); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/index.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/index.ts index ad09aee55e..6f23d36217 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/index.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/index.ts @@ -250,13 +250,14 @@ export function deployDevice(): RequestHandler { }; const connection = getDBConnection(req['keycloak_token']); - const surveyCritterService = new SurveyCritterService(connection); - const bctwService = new BctwService(user); try { await connection.open(); + const surveyCritterService = new SurveyCritterService(connection); await surveyCritterService.upsertDeployment(critterId, newDeploymentId); + + const bctwService = new BctwService(user); await bctwService.deployDevice(newDeploymentDevice); await connection.commit(); @@ -279,11 +280,15 @@ export function updateDeployment(): RequestHandler { username: req['system_user']?.user_identifier }; const critterId = Number(req.params.critterId); + const connection = getDBConnection(req['keycloak_token']); - const surveyCritterService = new SurveyCritterService(connection); - const bctw = new BctwService(user); + try { await connection.open(); + + const surveyCritterService = new SurveyCritterService(connection); + const bctw = new BctwService(user); + await surveyCritterService.upsertDeployment(critterId, req.body.deployment_id); await bctw.updateDeployment(req.body); await connection.commit(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/{bctwDeploymentId}.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/{bctwDeploymentId}.ts index cc68455851..d98ee990a1 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/{bctwDeploymentId}.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/deployments/{bctwDeploymentId}.ts @@ -113,14 +113,14 @@ export function deleteDeployment(): RequestHandler { const critterId = Number(req.params.critterId); const connection = getDBConnection(req['keycloak_token']); - const surveyCritterService = new SurveyCritterService(connection); - const bctwService = new BctwService(user); try { await connection.open(); + const surveyCritterService = new SurveyCritterService(connection); await surveyCritterService.removeDeployment(critterId, deploymentId); + const bctwService = new BctwService(user); await bctwService.deleteDeployment(deploymentId); await connection.commit(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/telemetry.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/telemetry.ts index feee2a63f9..7fbf63de05 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/telemetry.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/{critterId}/telemetry.ts @@ -266,11 +266,11 @@ export function getCritterTelemetry(): RequestHandler { const surveyId = Number(req.params.surveyId); const connection = getDBConnection(req['keycloak_token']); - const surveyCritterService = new SurveyCritterService(connection); - const bctwService = new BctwService(user); try { await connection.open(); + + const surveyCritterService = new SurveyCritterService(connection); const surveyCritters = await surveyCritterService.getCrittersInSurvey(surveyId); const critter = surveyCritters.find((surveyCritter) => surveyCritter.critter_id === critterId); @@ -281,8 +281,8 @@ export function getCritterTelemetry(): RequestHandler { const startDate = new Date(String(req.query.startDate)); const endDate = new Date(String(req.query.endDate)); + const bctwService = new BctwService(user); const points = await bctwService.getCritterTelemetryPoints(critter.critterbase_critter_id, startDate, endDate); - const tracks = await bctwService.getCritterTelemetryTracks(critter.critterbase_critter_id, startDate, endDate); await connection.commit(); diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/deployments.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/deployments.ts index 5f165530a1..211a072412 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/deployments.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/deployments.ts @@ -94,12 +94,12 @@ export function getDeploymentsInSurvey(): RequestHandler { const surveyId = Number(req.params.surveyId); const connection = getDBConnection(req['keycloak_token']); - const surveyCritterService = new SurveyCritterService(connection); - const bctwService = new BctwService(user); - try { await connection.open(); + const surveyCritterService = new SurveyCritterService(connection); + const bctwService = new BctwService(user); + const critter_ids = (await surveyCritterService.getCrittersInSurvey(surveyId)).map( (critter) => critter.critterbase_critter_id ); diff --git a/api/src/repositories/survey-critter-repository.test.ts b/api/src/repositories/survey-critter-repository.test.ts index 46e145c6f5..9969e53269 100644 --- a/api/src/repositories/survey-critter-repository.test.ts +++ b/api/src/repositories/survey-critter-repository.test.ts @@ -26,16 +26,16 @@ describe('SurveyRepository', () => { }); }); - describe('addCritterToSurvey', () => { + describe('addCrittersToSurvey', () => { it('should return result', async () => { - const mockResponse = { rows: [{ submissionId: 1 }], rowCount: 1 } as any as Promise>; + const mockResponse = { rows: [{ critter_id: 1 }], rowCount: 1 } as any as Promise>; const dbConnection = getMockDBConnection({ knex: () => mockResponse }); const repository = new SurveyCritterRepository(dbConnection); - const response = await repository.addCritterToSurvey(1, 'critter_id'); + const response = await repository.addCrittersToSurvey(1, ['critter_id']); - expect(response).to.be.undefined; + expect(response).to.be.deep.equal([1]); }); }); diff --git a/api/src/repositories/survey-critter-repository.ts b/api/src/repositories/survey-critter-repository.ts index 9f8b630583..2da82b9cb8 100644 --- a/api/src/repositories/survey-critter-repository.ts +++ b/api/src/repositories/survey-critter-repository.ts @@ -147,21 +147,21 @@ export class SurveyCritterRepository extends BaseRepository { * Add critter to survey * * @param {number} surveyId - * @param {string} critterId + * @param {string[]} critterIds * @return {*} {Promise} * @memberof SurveyCritterRepository */ - async addCritterToSurvey(surveyId: number, critterId: string): Promise { - defaultLog.debug({ label: 'addCritterToSurvey', surveyId }); + async addCrittersToSurvey(surveyId: number, critterIds: string[]): Promise { + defaultLog.debug({ label: 'addCrittersToSurvey', surveyId }); const queryBuilder = getKnex() .table('critter') - .insert({ survey_id: surveyId, critterbase_critter_id: critterId }) + .insert(critterIds.map((critterId) => ({ survey_id: surveyId, critterbase_critter_id: critterId }))) .returning('critter_id'); const response = await this.connection.knex(queryBuilder); - return response.rows[0].critter_id; + return response.rows.map((row) => row.critter_id); } /** diff --git a/api/src/services/critterbase-service.test.ts b/api/src/services/critterbase-service.test.ts index 169ad18839..e2ab4bc68b 100644 --- a/api/src/services/critterbase-service.test.ts +++ b/api/src/services/critterbase-service.test.ts @@ -2,7 +2,7 @@ import { AxiosResponse } from 'axios'; import chai, { expect } from 'chai'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import { CritterbaseService, ICritter } from './critterbase-service'; +import { CritterbaseService, ICreateCritter } from './critterbase-service'; import { KeycloakService } from './keycloak-service'; chai.use(sinonChai); @@ -145,12 +145,11 @@ describe('CritterbaseService', () => { describe('createCritter', () => { it('should create a critter', async () => { - const data: ICritter = { + const data: ICreateCritter = { wlh_id: 'aaaa', animal_id: 'aaaa', sex: 'male', itis_tsn: 1, - itis_scientific_name: 'Name', critter_comment: 'None.' }; const axiosStub = sinon.stub(cb.axiosInstance, 'post').resolves({ data: [] }); diff --git a/api/src/services/critterbase-service.ts b/api/src/services/critterbase-service.ts index 240aa1b74e..4f736139f0 100644 --- a/api/src/services/critterbase-service.ts +++ b/api/src/services/critterbase-service.ts @@ -16,13 +16,22 @@ export interface QueryParam { } export interface ICritter { - critter_id?: string; + critter_id: string; wlh_id: string | null; - animal_id: string; + animal_id: string | null; sex: string; itis_tsn: number; itis_scientific_name: string; - critter_comment: string; + critter_comment: string | null; +} + +export interface ICreateCritter { + critter_id?: string; + wlh_id?: string | null; + animal_id: string; // NOTE: In critterbase this is optional. For SIMS it should be required. + sex: string; + itis_tsn: number; + critter_comment?: string | null; } export interface ICapture { @@ -113,15 +122,48 @@ export interface ICollection { } export interface IBulkCreate { - critters: ICritter[]; - captures: ICapture[]; - collections: ICollection[]; - mortalities: IMortality[]; - locations: ILocation[]; - markings: IMarking[]; - quantitative_measurements: IQuantMeasurement[]; - qualitative_measurements: IQualMeasurement[]; - families: IFamilyPayload[]; + critters?: ICreateCritter[]; + captures?: ICapture[]; + collections?: ICollection[]; + mortalities?: IMortality[]; + locations?: ILocation[]; + markings?: IMarking[]; + quantitative_measurements?: IQuantMeasurement[]; + qualitative_measurements?: IQualMeasurement[]; + families?: IFamilyPayload[]; +} + +interface IBulkResponse { + critters: number; + captures: number; + collections: number; + mortalities: number; + locations: number; + markings: number; + quantitative_measurements: number; + qualitative_measurements: number; + families: number; + family_parents: number; + family_chidren: number; +} + +export interface IBulkCreateResponse { + created: IBulkResponse; +} + +export interface ICollectionUnitWithCategory { + collection_unit_id: string; + collection_category_id: string; + category_name: string; + unit_name: string; + description: string | null; +} + +export interface ICollectionCategory { + collection_category_id: string; + category_name: string; + description: string | null; + itis_tsn: number; } /** @@ -369,7 +411,7 @@ export class CritterbaseService { return this._makeGetRequest(`${CRITTER_ENDPOINT}/${critter_id}`, [{ key: 'format', value: 'detail' }]); } - async createCritter(data: ICritter) { + async createCritter(data: ICreateCritter) { const response = await this.axiosInstance.post(`${CRITTER_ENDPOINT}/create`, data); return response.data; } @@ -379,6 +421,11 @@ export class CritterbaseService { return response.data; } + async bulkCreate(data: IBulkCreate): Promise { + const response = await this.axiosInstance.post(BULK_ENDPOINT, data); + return response.data; + } + async getMultipleCrittersByIds(critter_ids: string[]): Promise { const response = await this.axiosInstance.post(CRITTER_ENDPOINT, { critter_ids }); return response.data; @@ -422,4 +469,30 @@ export class CritterbaseService { return data; } + + /** + * Find collection categories by tsn. Includes hierarchies. + * + * @async + * @param {string} tsn - ITIS TSN + * @returns {Promise} Collection categories + */ + async findTaxonCollectionCategories(tsn: string): Promise { + const response = await this.axiosInstance.get(`/xref/taxon-collection-categories?tsn=${tsn}`); + + return response.data; + } + + /** + * Find collection units by tsn. Includes hierarchies. + * + * @async + * @param {string} tsn - ITIS TSN + * @returns {Promise} Collection units + */ + async findTaxonCollectionUnits(tsn: string): Promise { + const response = await this.axiosInstance.get(`/xref/taxon-collection-units?tsn=${tsn}`); + + return response.data; + } } diff --git a/api/src/services/import-services/import-critters-service.interface.ts b/api/src/services/import-services/import-critters-service.interface.ts new file mode 100644 index 0000000000..0a99524824 --- /dev/null +++ b/api/src/services/import-services/import-critters-service.interface.ts @@ -0,0 +1,42 @@ +/** + * Type wrapper for unknown CSV rows/records + * + */ +export type Row = Record; + +/** + * A validated CSV Critter object + * + */ +export type CsvCritter = { + critter_id: string; + sex: string; + itis_tsn: number; + animal_id: string; + wlh_id?: string; + critter_comment?: string; +} & { + [collectionUnitColumn: string]: unknown; +}; + +/** + * Invalidated CSV Critter object + * + */ +export type PartialCsvCritter = Partial & { critter_id: string }; + +export type ValidationError = { row: number; message: string }; + +/** + * Conditional validation type similar to Zod SafeParseReturn + * + */ +export type Validation = + | { + success: true; + data: CsvCritter[]; + } + | { + success: false; + errors: ValidationError[]; + }; diff --git a/api/src/services/import-services/import-critters-service.test.ts b/api/src/services/import-services/import-critters-service.test.ts new file mode 100644 index 0000000000..338e347a55 --- /dev/null +++ b/api/src/services/import-services/import-critters-service.test.ts @@ -0,0 +1,783 @@ +import chai, { expect } from 'chai'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { WorkSheet } from 'xlsx'; +import { MediaFile } from '../../utils/media/media-file'; +import { critterStandardColumnValidator } from '../../utils/xlsx-utils/column-cell-utils'; +import * as xlsxUtils from '../../utils/xlsx-utils/worksheet-utils'; +import { getMockDBConnection } from '../../__mocks__/db'; +import { IBulkCreateResponse } from '../critterbase-service'; +import { ImportCrittersService } from './import-critters-service'; +import { CsvCritter, PartialCsvCritter } from './import-critters-service.interface'; + +chai.use(sinonChai); + +const mockConnection = getMockDBConnection(); + +describe('ImportCrittersService', () => { + describe('_getCritterRowsToValidate', () => { + it('it should correctly format rows', () => { + const rows = [ + { + SEX: 'Male', + ITIS_TSN: 1, + WLH_ID: '10-1000', + ALIAS: 'Carl', + COMMENT: 'Test', + COLLECTION: 'Unit', + BAD_COLLECTION: 'Bad' + } + ]; + const service = new ImportCrittersService(mockConnection); + + const parsedRow = service._getCritterRowsToValidate(rows, ['COLLECTION', 'TEST'])[0]; + + expect(parsedRow.sex).to.be.eq('Male'); + expect(parsedRow.itis_tsn).to.be.eq(1); + expect(parsedRow.wlh_id).to.be.eq('10-1000'); + expect(parsedRow.animal_id).to.be.eq('Carl'); + expect(parsedRow.critter_comment).to.be.eq('Test'); + expect(parsedRow.COLLECTION).to.be.eq('Unit'); + expect(parsedRow.TEST).to.be.undefined; + expect(parsedRow.BAD_COLLECTION).to.be.undefined; + }); + }); + + describe('_getCritterFromRow', () => { + it('should get all critter properties', () => { + const row: any = { + critter_id: 'id', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + extra_property: 'test' + }; + const service = new ImportCrittersService(mockConnection); + + const critter = service._getCritterFromRow(row); + + expect(critter).to.be.eql({ + critter_id: 'id', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment' + }); + }); + }); + + describe('_getCollectionUnitsFromRow', () => { + it('should get all collection unit properties', () => { + const row: any = { + critter_id: 'id', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'ID1', + HERD: 'ID2' + }; + const service = new ImportCrittersService(mockConnection); + + const collectionUnits = service._getCollectionUnitsFromRow(row); + + expect(collectionUnits).to.be.deep.equal([ + { collection_unit_id: 'ID1', critter_id: 'id' }, + { collection_unit_id: 'ID2', critter_id: 'id' } + ]); + }); + }); + + describe('_getValidTsns', () => { + afterEach(() => { + sinon.restore(); + }); + + it('should return unique list of tsns', async () => { + const service = new ImportCrittersService(mockConnection); + + const mockWorksheet = {} as unknown as WorkSheet; + + const getRowsStub = sinon + .stub(service, '_getRows') + .returns([{ itis_tsn: 1 }, { itis_tsn: 2 }, { itis_tsn: 2 }] as any); + + const getTaxonomyStub = sinon.stub(service.platformService, 'getTaxonomyByTsns').resolves([ + { tsn: '1', scientificName: 'a' }, + { tsn: '2', scientificName: 'b' } + ]); + const tsns = await service._getValidTsns(mockWorksheet); + expect(getRowsStub).to.have.been.calledWith(mockWorksheet); + expect(getTaxonomyStub).to.have.been.calledWith(['1', '2']); + expect(tsns).to.deep.equal(['1', '2']); + }); + }); + + describe('_getCollectionUnitMap', () => { + afterEach(() => { + sinon.restore(); + }); + + const collectionUnitsA = [ + { + collection_unit_id: '1', + collection_category_id: '2', + category_name: 'COLLECTION', + unit_name: 'UNIT_A', + description: 'description' + }, + { + collection_unit_id: '2', + collection_category_id: '3', + category_name: 'COLLECTION', + unit_name: 'UNIT_B', + description: 'description' + } + ]; + + const collectionUnitsB = [ + { + collection_unit_id: '1', + collection_category_id: '2', + category_name: 'HERD', + unit_name: 'UNIT_A', + description: 'description' + }, + { + collection_unit_id: '2', + collection_category_id: '3', + category_name: 'HERD', + unit_name: 'UNIT_B', + description: 'description' + } + ]; + + it('should return collection unit mapping', async () => { + const service = new ImportCrittersService(mockConnection); + + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const mockWorksheet = {} as unknown as WorkSheet; + + const findCollectionUnitsStub = sinon.stub(service.critterbaseService, 'findTaxonCollectionUnits'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + findCollectionUnitsStub.onCall(0).resolves(collectionUnitsA); + findCollectionUnitsStub.onCall(1).resolves(collectionUnitsB); + + const mapping = await service._getCollectionUnitMap(mockWorksheet, ['1', '2']); + expect(getColumnsStub).to.have.been.calledWith(mockWorksheet); + expect(findCollectionUnitsStub).to.have.been.calledTwice; + + expect(mapping).to.be.instanceof(Map); + expect(mapping.get('COLLECTION')).to.be.deep.equal({ collectionUnits: collectionUnitsA, tsn: 1 }); + expect(mapping.get('HERD')).to.be.deep.equal({ collectionUnits: collectionUnitsB, tsn: 2 }); + }); + + it('should return empty map when no collection unit columns', async () => { + const service = new ImportCrittersService(mockConnection); + + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const mockWorksheet = {} as unknown as WorkSheet; + + const findCollectionUnitsStub = sinon.stub(service.critterbaseService, 'findTaxonCollectionUnits'); + getColumnsStub.returns([]); + + const mapping = await service._getCollectionUnitMap(mockWorksheet, ['1', '2']); + expect(getColumnsStub).to.have.been.calledWith(mockWorksheet); + expect(findCollectionUnitsStub).to.have.not.been.called; + + expect(mapping).to.be.instanceof(Map); + }); + }); + + describe('_insertCsvCrittersIntoSimsAndCritterbase', () => { + afterEach(() => { + sinon.restore(); + }); + + const critters: CsvCritter[] = [ + { + critter_id: '1', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'Collection Unit' + }, + { + critter_id: '2', + sex: 'Female', + itis_tsn: 2, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + HERD: 'Herd Unit' + } + ]; + + it('should correctly parse collection units and critters and insert into sims / critterbase', async () => { + const service = new ImportCrittersService(mockConnection); + + const critterbaseBulkCreateStub = sinon.stub(service.critterbaseService, 'bulkCreate'); + const simsAddSurveyCrittersStub = sinon.stub(service.surveyCritterService, 'addCrittersToSurvey'); + + critterbaseBulkCreateStub.resolves({ created: { critters: 2, collections: 1 } } as IBulkCreateResponse); + simsAddSurveyCrittersStub.resolves([1]); + + const ids = await service._insertCsvCrittersIntoSimsAndCritterbase(1, critters); + + expect(critterbaseBulkCreateStub).to.have.been.calledWithExactly({ + critters: [ + { + critter_id: '1', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment' + }, + { + critter_id: '2', + sex: 'Female', + itis_tsn: 2, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment' + } + ], + collections: [ + { collection_unit_id: 'Collection Unit', critter_id: '1' }, + { collection_unit_id: 'Herd Unit', critter_id: '2' } + ] + }); + + expect(ids).to.be.deep.equal([1]); + }); + + it('should throw error if response from critterbase is less than provided critters', async () => { + const service = new ImportCrittersService(mockConnection); + + const critterbaseBulkCreateStub = sinon.stub(service.critterbaseService, 'bulkCreate'); + const simsAddSurveyCrittersStub = sinon.stub(service.surveyCritterService, 'addCrittersToSurvey'); + + critterbaseBulkCreateStub.resolves({ created: { critters: 1, collections: 1 } } as IBulkCreateResponse); + simsAddSurveyCrittersStub.resolves([1]); + + try { + await service._insertCsvCrittersIntoSimsAndCritterbase(1, critters); + expect.fail(); + } catch (err: any) { + expect(err.message).to.be.equal('Unable to fully import critters from CSV'); + } + + expect(simsAddSurveyCrittersStub).to.not.have.been.called; + }); + }); + + describe('_validateRows', () => { + afterEach(() => { + sinon.restore(); + }); + + const collectionUnitsA = [ + { + collection_unit_id: '1', + collection_category_id: '2', + category_name: 'COLLECTION', + unit_name: 'UNIT_A', + description: 'description' + }, + { + collection_unit_id: '2', + collection_category_id: '3', + category_name: 'COLLECTION', + unit_name: 'UNIT_B', + description: 'description' + } + ]; + + const collectionUnitsB = [ + { + collection_unit_id: '1', + collection_category_id: '2', + category_name: 'HERD', + unit_name: 'UNIT_A', + description: 'description' + }, + { + collection_unit_id: '2', + collection_category_id: '3', + category_name: 'HERD', + unit_name: 'UNIT_B', + description: 'description' + } + ]; + + it('should return successful', async () => { + const service = new ImportCrittersService(mockConnection); + const getRowsStub = sinon.stub(service, '_getRows'); + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); + const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); + const collectionMapStub = sinon.stub(service, '_getCollectionUnitMap'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + surveyAliasesStub.resolves(new Set(['Not Carl', 'Carlita'])); + getValidTsnsStub.resolves(['1', '2']); + collectionMapStub.resolves( + new Map([ + ['COLLECTION', { collectionUnits: collectionUnitsA, tsn: 1 }], + ['HERD', { collectionUnits: collectionUnitsB, tsn: 2 }] + ]) + ); + + const validRows: CsvCritter[] = [ + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + }, + { + critter_id: 'B', + sex: 'Male', + itis_tsn: 2, + animal_id: 'Test', + wlh_id: '10-1000', + critter_comment: 'comment', + HERD: 'UNIT_B' + } + ]; + + getRowsStub.returns(validRows); + + const validation = await service._validateRows(1, {} as WorkSheet); + + expect(validation.success).to.be.true; + + if (validation.success) { + expect(validation.data).to.be.deep.equal([ + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: '1' + }, + { + critter_id: 'B', + sex: 'Male', + itis_tsn: 2, + animal_id: 'Test', + wlh_id: '10-1000', + critter_comment: 'comment', + HERD: '2' + } + ]); + } + }); + + it('should push error when sex is undefined or invalid', async () => { + const service = new ImportCrittersService(mockConnection); + const getRowsStub = sinon.stub(service, '_getRows'); + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); + const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); + const collectionMapStub = sinon.stub(service, '_getCollectionUnitMap'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + surveyAliasesStub.resolves(new Set(['Not Carl', 'Carlita'])); + getValidTsnsStub.resolves(['1', '2']); + collectionMapStub.resolves( + new Map([ + ['COLLECTION', { collectionUnits: collectionUnitsA, tsn: 1 }], + ['HERD', { collectionUnits: collectionUnitsB, tsn: 2 }] + ]) + ); + + const invalidRows: PartialCsvCritter[] = [ + { + critter_id: 'A', + sex: undefined, + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + }, + { + critter_id: 'A', + sex: 'Whoops' as any, + itis_tsn: 1, + animal_id: 'Carl2', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + } + ]; + + getRowsStub.returns(invalidRows); + + const validation = await service._validateRows(1, {} as WorkSheet); + + expect(validation.success).to.be.false; + if (!validation.success) { + expect(validation.errors.length).to.be.eq(2); + expect(validation.errors).to.be.deep.equal([ + { + message: 'Invalid SEX. Expecting: UNKNOWN, MALE, FEMALE, HERMAPHRODITIC.', + row: 0 + }, + { + message: 'Invalid SEX. Expecting: UNKNOWN, MALE, FEMALE, HERMAPHRODITIC.', + row: 1 + } + ]); + } + }); + + it('should push error when wlh_id is invalid regex / shape', async () => { + const service = new ImportCrittersService(mockConnection); + const getRowsStub = sinon.stub(service, '_getRows'); + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); + const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); + const collectionMapStub = sinon.stub(service, '_getCollectionUnitMap'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + surveyAliasesStub.resolves(new Set(['Not Carl', 'Carlita'])); + getValidTsnsStub.resolves(['1', '2']); + collectionMapStub.resolves( + new Map([ + ['COLLECTION', { collectionUnits: collectionUnitsA, tsn: 1 }], + ['HERD', { collectionUnits: collectionUnitsB, tsn: 2 }] + ]) + ); + + const invalidRows: PartialCsvCritter[] = [ + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '101000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + }, + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl2', + wlh_id: '1-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + } + ]; + + getRowsStub.returns(invalidRows); + + const validation = await service._validateRows(1, {} as WorkSheet); + + expect(validation.success).to.be.false; + if (!validation.success) { + expect(validation.errors.length).to.be.eq(2); + expect(validation.errors).to.be.deep.equal([ + { + message: `Invalid WLH_ID. Example format '10-1000R'.`, + row: 0 + }, + { + message: `Invalid WLH_ID. Example format '10-1000R'.`, + row: 1 + } + ]); + } + }); + + it('should push error when itis_tsn undefined or invalid option', async () => { + const service = new ImportCrittersService(mockConnection); + const getRowsStub = sinon.stub(service, '_getRows'); + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); + const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); + const collectionMapStub = sinon.stub(service, '_getCollectionUnitMap'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + surveyAliasesStub.resolves(new Set(['Not Carl', 'Carlita'])); + getValidTsnsStub.resolves(['1', '2']); + collectionMapStub.resolves( + new Map([ + ['COLLECTION', { collectionUnits: collectionUnitsA, tsn: 1 }], + ['HERD', { collectionUnits: collectionUnitsB, tsn: 2 }] + ]) + ); + + const invalidRows: PartialCsvCritter[] = [ + { + critter_id: 'A', + sex: 'Male', + itis_tsn: undefined, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + }, + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 10, + animal_id: 'Carl2', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + } + ]; + + getRowsStub.returns(invalidRows); + + const validation = await service._validateRows(1, {} as WorkSheet); + + expect(validation.success).to.be.false; + if (!validation.success) { + expect(validation.errors.length).to.be.eq(4); + expect(validation.errors).to.be.deep.equal([ + { + message: `Invalid ITIS_TSN.`, + row: 0 + }, + { + message: `Invalid COLLECTION. Cell value not allowed for TSN.`, + row: 0 + }, + { + message: `Invalid ITIS_TSN.`, + row: 1 + }, + { + message: `Invalid COLLECTION. Cell value not allowed for TSN.`, + row: 1 + } + ]); + } + }); + + it('should push error when itis_tsn undefined or invalid option', async () => { + const service = new ImportCrittersService(mockConnection); + const getRowsStub = sinon.stub(service, '_getRows'); + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); + const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); + const collectionMapStub = sinon.stub(service, '_getCollectionUnitMap'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + surveyAliasesStub.resolves(new Set(['Not Carl', 'Carlita'])); + getValidTsnsStub.resolves(['1', '2']); + collectionMapStub.resolves( + new Map([ + ['COLLECTION', { collectionUnits: collectionUnitsA, tsn: 1 }], + ['HERD', { collectionUnits: collectionUnitsB, tsn: 2 }] + ]) + ); + + const invalidRows: PartialCsvCritter[] = [ + { + critter_id: 'A', + sex: 'Male', + itis_tsn: undefined, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + }, + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 10, + animal_id: 'Carl2', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + } + ]; + + getRowsStub.returns(invalidRows); + + const validation = await service._validateRows(1, {} as WorkSheet); + + expect(validation.success).to.be.false; + if (!validation.success) { + expect(validation.errors.length).to.be.eq(4); + expect(validation.errors).to.be.deep.equal([ + { + message: `Invalid ITIS_TSN.`, + row: 0 + }, + { + message: `Invalid COLLECTION. Cell value not allowed for TSN.`, + row: 0 + }, + { + message: `Invalid ITIS_TSN.`, + row: 1 + }, + { + message: `Invalid COLLECTION. Cell value not allowed for TSN.`, + row: 1 + } + ]); + } + }); + + it('should push error if alias undefined, duplicate or exists in survey', async () => { + const service = new ImportCrittersService(mockConnection); + const getRowsStub = sinon.stub(service, '_getRows'); + const getColumnsStub = sinon.stub(service, '_getNonStandardColumns'); + const surveyAliasesStub = sinon.stub(service.surveyCritterService, 'getUniqueSurveyCritterAliases'); + const getValidTsnsStub = sinon.stub(service, '_getValidTsns'); + const collectionMapStub = sinon.stub(service, '_getCollectionUnitMap'); + + getColumnsStub.returns(['COLLECTION', 'HERD']); + surveyAliasesStub.resolves(new Set(['Not Carl', 'Carlita'])); + getValidTsnsStub.resolves(['1', '2']); + collectionMapStub.resolves( + new Map([ + ['COLLECTION', { collectionUnits: collectionUnitsA, tsn: 1 }], + ['HERD', { collectionUnits: collectionUnitsB, tsn: 2 }] + ]) + ); + + const invalidRows: PartialCsvCritter[] = [ + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + }, + { + critter_id: 'A', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carlita', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'UNIT_A' + } + ]; + + getRowsStub.returns(invalidRows); + + const validation = await service._validateRows(1, {} as WorkSheet); + + expect(validation.success).to.be.false; + if (!validation.success) { + expect(validation.errors.length).to.be.eq(1); + expect(validation.errors).to.be.deep.equal([ + { + message: `Invalid ALIAS. Must be unique in Survey and CSV.`, + row: 1 + } + ]); + } + }); + }); + + describe('_validate', () => { + afterEach(() => { + sinon.restore(); + }); + + it('should throw error when csv validation fails', async () => { + const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile').returns(false); + + const service = new ImportCrittersService(mockConnection); + + sinon.stub(service, '_validateRows'); + + try { + await service._validate(1, {} as WorkSheet); + expect.fail(); + } catch (err: any) { + expect(err.message).to.contain('Column validator failed.'); + } + expect(validateCsvStub).to.have.been.calledOnceWithExactly({}, critterStandardColumnValidator); + }); + + it('should call _validateRows if csv validation succeeds', async () => { + const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile'); + + const service = new ImportCrittersService(mockConnection); + + const validateRowsStub = sinon.stub(service, '_validateRows'); + + validateCsvStub.returns(true); + validateRowsStub.resolves({ success: true, data: [] }); + + const data = await service._validate(1, {} as WorkSheet); + expect(validateRowsStub).to.have.been.calledOnceWithExactly(1, {}); + expect(data).to.be.deep.equal([]); + }); + + it('should throw error if row validation fails', async () => { + const validateCsvStub = sinon.stub(xlsxUtils, 'validateCsvFile'); + + const service = new ImportCrittersService(mockConnection); + + const validateRowsStub = sinon.stub(service, '_validateRows'); + + validateCsvStub.returns(true); + validateRowsStub.resolves({ success: false, errors: [] }); + + try { + await service._validate(1, {} as WorkSheet); + + expect.fail(); + } catch (err: any) { + expect(err.message).to.contain('Failed to import Critter CSV.'); + } + expect(validateRowsStub).to.have.been.calledOnceWithExactly(1, {}); + }); + }); + + describe('import', () => { + it('should pass values to correct methods', async () => { + const service = new ImportCrittersService(mockConnection); + const csv = new MediaFile('file', 'mime', Buffer.alloc(1)); + + const critter: CsvCritter = { + critter_id: 'id', + sex: 'Male', + itis_tsn: 1, + animal_id: 'Carl', + wlh_id: '10-1000', + critter_comment: 'comment', + COLLECTION: 'Unit' + }; + + const getWorksheetStub = sinon.stub(service, '_getWorksheet').returns({} as unknown as WorkSheet); + const validateStub = sinon.stub(service, '_validate').resolves([critter]); + const insertStub = sinon.stub(service, '_insertCsvCrittersIntoSimsAndCritterbase').resolves([1]); + + const data = await service.import(1, csv); + + expect(getWorksheetStub).to.have.been.calledWithExactly(csv); + expect(validateStub).to.have.been.calledWithExactly(1, {}); + expect(insertStub).to.have.been.calledWithExactly(1, [critter]); + + expect(data).to.be.deep.equal([1]); + }); + }); +}); diff --git a/api/src/services/import-services/import-critters-service.ts b/api/src/services/import-services/import-critters-service.ts new file mode 100644 index 0000000000..0c7a97550c --- /dev/null +++ b/api/src/services/import-services/import-critters-service.ts @@ -0,0 +1,414 @@ +import { capitalize, keys, omit, toUpper, uniq } from 'lodash'; +import { v4 as uuid } from 'uuid'; +import { WorkSheet } from 'xlsx'; +import { IDBConnection } from '../../database/db'; +import { ApiGeneralError } from '../../errors/api-error'; +import { getLogger } from '../../utils/logger'; +import { MediaFile } from '../../utils/media/media-file'; +import { + critterStandardColumnValidator, + getAliasFromRow, + getColumnValidatorSpecification, + getDescriptionFromRow, + getSexFromRow, + getTsnFromRow, + getWlhIdFromRow +} from '../../utils/xlsx-utils/column-cell-utils'; +import { + constructXLSXWorkbook, + getDefaultWorksheet, + getNonStandardColumnNamesFromWorksheet, + getWorksheetRowObjects, + validateCsvFile +} from '../../utils/xlsx-utils/worksheet-utils'; +import { + CritterbaseService, + IBulkCreate, + ICollection, + ICollectionUnitWithCategory, + ICreateCritter +} from '../critterbase-service'; +import { DBService } from '../db-service'; +import { PlatformService } from '../platform-service'; +import { SurveyCritterService } from '../survey-critter-service'; +import { CsvCritter, PartialCsvCritter, Row, Validation, ValidationError } from './import-critters-service.interface'; + +const defaultLog = getLogger('services/import/import-critters-service'); + +const CSV_CRITTER_SEX_OPTIONS = ['UNKNOWN', 'MALE', 'FEMALE', 'HERMAPHRODITIC']; + +/** + * + * @class ImportCrittersService + * @extends DBService + * + */ +export class ImportCrittersService extends DBService { + platformService: PlatformService; + critterbaseService: CritterbaseService; + surveyCritterService: SurveyCritterService; + + _rows?: PartialCsvCritter[]; + + constructor(connection: IDBConnection) { + super(connection); + + this.platformService = new PlatformService(connection); + this.surveyCritterService = new SurveyCritterService(connection); + this.critterbaseService = new CritterbaseService({ + keycloak_guid: connection.systemUserGUID(), + username: connection.systemUserIdentifier() + }); + } + + /** + * Get the worksheet from the CSV file. + * + * @param {MediaFile} critterCsv - CSV MediaFile + * @returns {WorkSheet} Xlsx worksheet + */ + _getWorksheet(critterCsv: MediaFile) { + return getDefaultWorksheet(constructXLSXWorkbook(critterCsv)); + } + + /** + * Get non-standard columns (collection unit columns) from worksheet. + * + * @param {WorkSheet} worksheet - Xlsx worksheet + * @returns {string[]} Array of non-standard headers from CSV (worksheet) + */ + _getNonStandardColumns(worksheet: WorkSheet) { + return uniq(getNonStandardColumnNamesFromWorksheet(worksheet, critterStandardColumnValidator)); + } + /** + * Parse the CSV rows into the Critterbase critter format. + * + * @param {Row[]} rows - CSV rows + * @returns {PartialCsvCritter[]} CSV critters before validation + */ + _getCritterRowsToValidate(rows: Row[], collectionUnitColumns: string[]): PartialCsvCritter[] { + return rows.map((row) => { + // Standard critter properties from CSV + const standardCritterRow: PartialCsvCritter = { + critter_id: uuid(), // Generate a uuid for each critter for convienence + sex: getSexFromRow(row), + itis_tsn: getTsnFromRow(row), + wlh_id: getWlhIdFromRow(row), + animal_id: getAliasFromRow(row), + critter_comment: getDescriptionFromRow(row) + }; + + // All other properties must be collection units ie: `population unit` or `herd unit` etc... + collectionUnitColumns.forEach((categoryHeader) => { + standardCritterRow[categoryHeader] = row[categoryHeader]; + }); + + return standardCritterRow; + }); + } + + /** + * Get the critter rows from the xlsx worksheet. + * + * @param {WorkSheet} worksheet + * @returns {PartialCsvCritter[]} List of partial CSV Critters + */ + _getRows(worksheet: WorkSheet) { + // Attempt to retrieve from rows property to prevent unnecessary parsing + if (this._rows) { + return this._rows; + } + + // Convert the worksheet into an array of records + const worksheetRows = getWorksheetRowObjects(worksheet); + + // Get the collection unit columns (all non standard columns) + const collectionUnitColumns = this._getNonStandardColumns(worksheet); + + // Pre parse the records into partial critter rows + this._rows = this._getCritterRowsToValidate(worksheetRows, collectionUnitColumns); + + return this._rows; + } + + /** + * Get critter from properties from row. + * + * @param {CsvCritter} row - Row object as CsvCritter + * @returns {ICreateCritter} Create critter object + */ + _getCritterFromRow(row: CsvCritter): ICreateCritter { + return { + critter_id: row.critter_id, + sex: capitalize(row.sex), + itis_tsn: row.itis_tsn, + animal_id: row.animal_id, + wlh_id: row.wlh_id, + critter_comment: row.critter_comment + }; + } + + /** + * Get list of collection units from row. + * + * @param {CsvCritter} row - Row object as a CsvCritter + * @returns {ICollection[]} Array of collection units + */ + _getCollectionUnitsFromRow(row: CsvCritter): ICollection[] { + const critterId = row.critter_id; + + // Get portion of row object that is not a critter + const partialRow = omit(row, keys(this._getCritterFromRow(row))); + + // Keys of collection units + const collectionUnitKeys = keys(partialRow); + + // Return an array of formatted collection units for bulk create + return collectionUnitKeys + .filter((key) => partialRow[key]) + .map((key) => ({ collection_unit_id: partialRow[key], critter_id: critterId })); + } + + /** + * Get a Set of valid ITIS TSNS from xlsx worksheet. + * + * @async + * @param {WorkSheet} worksheet - Xlsx Worksheet + * @returns {Promise} Unique Set of valid TSNS from worksheet. + */ + async _getValidTsns(worksheet: WorkSheet): Promise { + const rows = this._getRows(worksheet); + + // Get a unique list of tsns from worksheet + const critterTsns = uniq(rows.map((row) => String(row.itis_tsn))); + + // Query the platform service (taxonomy) for matching tsns + const taxonomy = await this.platformService.getTaxonomyByTsns(critterTsns); + + return taxonomy.map((taxon) => taxon.tsn); + } + + /** + * Get a mapping of collection units for a list of tsns. + * Used in the zod validation. + * + * @example new Map([['Population Unit', new Set(['Atlin', 'Unit B'])]]); + * + * @async + * @param {WorkSheet} worksheet - Xlsx Worksheet + * @param {string[]} tsns - List of unique and valid TSNS + * @returns {Promise} Collection unit mapping + */ + async _getCollectionUnitMap(worksheet: WorkSheet, tsns: string[]) { + const collectionUnitMap = new Map(); + + const collectionUnitColumns = this._getNonStandardColumns(worksheet); + + // If no collection unit columns return empty Map + if (!collectionUnitColumns.length) { + return collectionUnitMap; + } + + // Get the collection units for all the tsns in the worksheet + const tsnCollectionUnits = await Promise.all( + tsns.map((tsn) => this.critterbaseService.findTaxonCollectionUnits(tsn)) + ); + + tsnCollectionUnits.forEach((collectionUnits, index) => { + if (collectionUnits.length) { + collectionUnitMap.set(toUpper(collectionUnits[0].category_name), { collectionUnits, tsn: Number(tsns[index]) }); + } + }); + + return collectionUnitMap; + } + + /** + * Validate CSV worksheet rows against reference data. + * + * @async + * @param {number} surveyId - Survey identifier + * @param {WorkSheet} worksheet - Xlsx worksheet + * @returns {Promise} Conditional validation object + */ + async _validateRows(surveyId: number, worksheet: WorkSheet): Promise { + const rows = this._getRows(worksheet); + const nonStandardColumns = this._getNonStandardColumns(worksheet); + + // Retrieve the dynamic validation config + const [validRowTsns, surveyCritterAliases] = await Promise.all([ + this._getValidTsns(worksheet), + this.surveyCritterService.getUniqueSurveyCritterAliases(surveyId) + ]); + const collectionUnitMap = await this._getCollectionUnitMap(worksheet, validRowTsns); + + // Parse reference data for validation + const tsnSet = new Set(validRowTsns.map((tsn) => Number(tsn))); + const csvCritterAliases = rows.map((row) => row.animal_id); + + // Track the row validation errors + const errors: ValidationError[] = []; + + const csvCritters = rows.map((row, index) => { + /** + * -------------------------------------------------------------------- + * STANDARD ROW VALIDATION + * -------------------------------------------------------------------- + */ + + // SEX is a required property and must be a correct value + const invalidSex = !row.sex || !CSV_CRITTER_SEX_OPTIONS.includes(toUpper(row.sex)); + // WLH_ID must follow regex pattern + const invalidWlhId = row.wlh_id && !/^\d{2}-.+/.exec(row.wlh_id); + // ITIS_TSN is required and be a valid TSN + const invalidTsn = !row.itis_tsn || !tsnSet.has(row.itis_tsn); + // ALIAS is required and must not already exist in Survey or CSV + const invalidAlias = + !row.animal_id || + surveyCritterAliases.has(row.animal_id) || + csvCritterAliases.filter((value) => value === row.animal_id).length > 1; + + if (invalidSex) { + errors.push({ row: index, message: `Invalid SEX. Expecting: ${CSV_CRITTER_SEX_OPTIONS.join(', ')}.` }); + } + if (invalidWlhId) { + errors.push({ row: index, message: `Invalid WLH_ID. Example format '10-1000R'.` }); + } + if (invalidTsn) { + errors.push({ row: index, message: `Invalid ITIS_TSN.` }); + } + if (invalidAlias) { + errors.push({ row: index, message: `Invalid ALIAS. Must be unique in Survey and CSV.` }); + } + + /** + * -------------------------------------------------------------------- + * NON-STANDARD ROW VALIDATION + * -------------------------------------------------------------------- + */ + + nonStandardColumns.forEach((column) => { + const collectionUnitColumn = collectionUnitMap.get(column); + // Remove property if undefined or not a collection unit + if (!collectionUnitColumn || !row[column]) { + delete row[column]; + return; + } + + // Attempt to find the collection unit with the cell value from the mapping + const collectionUnitMatch = collectionUnitColumn.collectionUnits.find( + (unit) => unit.unit_name.toLowerCase() === String(row[column]).toLowerCase() + ); + // Collection unit must be a valid value + if (!collectionUnitMatch) { + errors.push({ row: index, message: `Invalid ${column}. Cell value is not valid.` }); + } + // Collection unit must have correct TSN mapping + else if (row.itis_tsn !== collectionUnitColumn.tsn) { + errors.push({ row: index, message: `Invalid ${column}. Cell value not allowed for TSN.` }); + } else { + // Update the cell to be the collection unit id + row[column] = collectionUnitMatch.collection_unit_id; + } + }); + + return row; + }); + + // If validation successful the rows should all be CsvCritters + if (!errors.length) { + return { success: true, data: csvCritters as CsvCritter[] }; + } + + return { success: false, errors }; + } + + /** + * Validate the worksheet contains no errors within the data or structure of CSV. + * + * @async + * @param {number} surveyId - Survey identifier + * @param {WorkSheet} worksheet - Xlsx worksheet + * @throws {ApiGeneralError} - If validation fails + * @returns {Promise} Validated CSV rows + */ + async _validate(surveyId: number, worksheet: WorkSheet): Promise { + // Validate the standard columns in the CSV file + if (!validateCsvFile(worksheet, critterStandardColumnValidator)) { + throw new ApiGeneralError(`Column validator failed. Column headers or cell data types are incorrect.`, [ + { column_specification: getColumnValidatorSpecification(critterStandardColumnValidator) }, + 'importCrittersService->_validate->validateCsvFile' + ]); + } + + // Validate the CSV rows with reference data + const validation = await this._validateRows(surveyId, worksheet); + + // Throw error is row validation failed and inject validation errors + if (!validation.success) { + throw new ApiGeneralError(`Failed to import Critter CSV. Column data validator failed.`, [ + { column_validation: validation.errors }, + 'importCrittersService->_validate->_validateRows' + ]); + } + + return validation.data; + } + + /** + * Insert CSV critters into Critterbase and SIMS. + * + * @async + * @param {number} surveyId - Survey identifier + * @param {CsvCritter[]} critterRows - CSV row critters + * @throws {ApiGeneralError} - If unable to fully insert records into Critterbase + * @returns {Promise} List of inserted survey critter ids + */ + async _insertCsvCrittersIntoSimsAndCritterbase(surveyId: number, critterRows: CsvCritter[]): Promise { + const simsPayload: string[] = []; + const critterbasePayload: IBulkCreate = { critters: [], collections: [] }; + + // Convert rows to Critterbase and SIMS payloads + for (const row of critterRows) { + simsPayload.push(row.critter_id); + critterbasePayload.critters?.push(this._getCritterFromRow(row)); + critterbasePayload.collections = critterbasePayload.collections?.concat(this._getCollectionUnitsFromRow(row)); + } + + defaultLog.debug({ label: 'critter import payloads', simsPayload, critterbasePayload }); + + // Add critters to Critterbase + const bulkResponse = await this.critterbaseService.bulkCreate(critterbasePayload); + + // Check critterbase inserted the full list of critters + // In reality this error should not be triggered, safeguard to prevent floating critter ids in SIMS + if (bulkResponse.created.critters !== simsPayload.length) { + throw new ApiGeneralError('Unable to fully import critters from CSV', [ + 'importCrittersService -> insertCsvCrittersIntoSimsAndCritterbase', + 'critterbase bulk create response count !== critterIds.length' + ]); + } + + // Add Critters to SIMS survey + return this.surveyCritterService.addCrittersToSurvey(surveyId, simsPayload); + } + + /** + * Import the CSV into SIMS and Critterbase. + * + * @async + * @param {number} surveyId - Survey identifier + * @param {MediaFile} critterCsv - CSV MediaFile + * @returns {Promise} List of survey critter identifiers + */ + async import(surveyId: number, critterCsv: MediaFile): Promise { + // Get the worksheet from the CSV + const worksheet = this._getWorksheet(critterCsv); + + // Validate the standard columns and the data of the CSV + const critters = await this._validate(surveyId, worksheet); + + // Insert the data into SIMS and Critterbase + return this._insertCsvCrittersIntoSimsAndCritterbase(surveyId, critters); + } +} diff --git a/api/src/services/observation-service.ts b/api/src/services/observation-service.ts index 720da3e757..5bd5200643 100644 --- a/api/src/services/observation-service.ts +++ b/api/src/services/observation-service.ts @@ -46,14 +46,14 @@ import { getDateFromRow, getLatitudeFromRow, getLongitudeFromRow, - getNonStandardColumnNamesFromWorksheet, getTimeFromRow, getTsnFromRow, observationStandardColumnValidator -} from '../utils/observation-xlsx-utils/standard-column-utils'; +} from '../utils/xlsx-utils/column-cell-utils'; import { constructXLSXWorkbook, getDefaultWorksheet, + getNonStandardColumnNamesFromWorksheet, getWorksheetRowObjects, validateCsvFile } from '../utils/xlsx-utils/worksheet-utils'; @@ -466,7 +466,10 @@ export class ObservationService extends DBService { } // Filter out the standard columns from the worksheet - const nonStandardColumnNames = getNonStandardColumnNamesFromWorksheet(xlsxWorksheet); + const nonStandardColumnNames = getNonStandardColumnNamesFromWorksheet( + xlsxWorksheet, + observationStandardColumnValidator + ); // Get the worksheet row objects const worksheetRowObjects = getWorksheetRowObjects(xlsxWorksheet); @@ -729,7 +732,7 @@ export class ObservationService extends DBService { * name to match its ITIS TSN. * * @template RecordWithTaxonFields - * @param {RecordWithTaxonFields[]} records + * @param {RecordWithTaxonFields[]} recordsToPatch * @return {*} {Promise} * @memberof ObservationService */ diff --git a/api/src/services/survey-critter-service.test.ts b/api/src/services/survey-critter-service.test.ts index 87c6093dc7..0a3230b75a 100644 --- a/api/src/services/survey-critter-service.test.ts +++ b/api/src/services/survey-critter-service.test.ts @@ -34,17 +34,17 @@ describe('SurveyService', () => { }); }); - describe('addCritterToSurvey', () => { + describe('addCrittersToSurvey', () => { it('returns the first row on success', async () => { const dbConnection = getMockDBConnection(); const service = new SurveyCritterService(dbConnection); - const repoStub = sinon.stub(SurveyCritterRepository.prototype, 'addCritterToSurvey').resolves(); + const repoStub = sinon.stub(SurveyCritterRepository.prototype, 'addCrittersToSurvey').resolves([1]); const response = await service.addCritterToSurvey(1, 'critter_id'); expect(repoStub).to.be.calledOnce; - expect(response).to.be.undefined; + expect(response).to.be.equal(1); }); }); diff --git a/api/src/services/survey-critter-service.ts b/api/src/services/survey-critter-service.ts index 6674774c4e..fc1ac5c142 100644 --- a/api/src/services/survey-critter-service.ts +++ b/api/src/services/survey-critter-service.ts @@ -21,12 +21,18 @@ export type FindCrittersResponse = Pick< */ export class SurveyCritterService extends DBService { critterRepository: SurveyCritterRepository; + critterbaseService: CritterbaseService; constructor(connection: IDBConnection) { super(connection); this.critterRepository = new SurveyCritterRepository(connection); + this.critterbaseService = new CritterbaseService({ + keycloak_guid: connection.systemUserGUID(), + username: connection.systemUserIdentifier() + }); } + /** * Get all critter associations for the given survey. This only gets you critter ids, which can be used to fetch * details from the external system. @@ -130,7 +136,21 @@ export class SurveyCritterService extends DBService { * @memberof SurveyCritterService */ async addCritterToSurvey(surveyId: number, critterBaseCritterId: string): Promise { - return this.critterRepository.addCritterToSurvey(surveyId, critterBaseCritterId); + const response = await this.critterRepository.addCrittersToSurvey(surveyId, [critterBaseCritterId]); + + return response[0]; + } + + /** + * Add multiple critters to a survey. Does not create anything in the external system. + * + * @param {number} surveyId + * @param {string} critterBaseCritterIds + * @return {*} {Promise} + * @memberof SurveyCritterService + */ + async addCrittersToSurvey(surveyId: number, critterBaseCritterIds: string[]): Promise { + return this.critterRepository.addCrittersToSurvey(surveyId, critterBaseCritterIds); } /** @@ -180,4 +200,23 @@ export class SurveyCritterService extends DBService { async removeDeployment(critterId: number, deploymentId: string): Promise { return this.critterRepository.removeDeployment(critterId, deploymentId); } + + /** + * Get unique Set of critter aliases (animal id / nickname) of a survey. + * + * @param {number} surveyId + * @return {*} {Promise} + * @memberof SurveyCritterService + */ + async getUniqueSurveyCritterAliases(surveyId: number): Promise> { + const surveyCritters = await this.getCrittersInSurvey(surveyId); + + const critterbaseCritterIds = surveyCritters.map((critter) => critter.critterbase_critter_id); + + const critters = await this.critterbaseService.getMultipleCrittersByIds(critterbaseCritterIds); + + // Return a unique Set of non-null critterbase aliases of a Survey + // Note: The type from filtered critters should be Set not Set + return new Set(critters.filter(Boolean).map((critter) => critter.animal_id)) as Set; + } } diff --git a/api/src/utils/observation-xlsx-utils/standard-column-utils.test.ts b/api/src/utils/observation-xlsx-utils/standard-column-utils.test.ts deleted file mode 100644 index 1ebda9d77a..0000000000 --- a/api/src/utils/observation-xlsx-utils/standard-column-utils.test.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { expect } from 'chai'; -import { describe } from 'mocha'; -import xlsx from 'xlsx'; - -import * as standard_column_utils from './standard-column-utils'; - -describe('standard-column-utils', () => { - describe('getNonStandardColumnNamesFromWorksheet', () => { - it('returns the non-standard column headers in UPPERCASE', () => { - const xlsxWorksheet: xlsx.WorkSheet = { - A1: { t: 's', v: 'Species' }, - B1: { t: 's', v: 'Count' }, - C1: { t: 's', v: 'Date' }, - D1: { t: 's', v: 'Time' }, - E1: { t: 's', v: 'Latitude' }, - F1: { t: 's', v: 'Longitude' }, - G1: { t: 's', v: 'Antler Configuration' }, - H1: { t: 's', v: 'Wind Direction' }, - A2: { t: 'n', w: '180703', v: 180703 }, - B2: { t: 'n', w: '1', v: 1 }, - C2: { z: 'm/d/yy', t: 'd', v: '1970-01-01T08:00:00.000Z', w: '1/1/70' }, - D2: { t: 's', v: '9:01' }, - E2: { t: 'n', w: '-58', v: -58 }, - F2: { t: 'n', w: '-123', v: -123 }, - G2: { t: 's', v: 'more than 3 points' }, - H2: { t: 's', v: 'North' }, - A3: { t: 'n', w: '180596', v: 180596 }, - B3: { t: 'n', w: '2', v: 2 }, - C3: { z: 'm/d/yy', t: 'd', v: '1970-01-01T08:00:00.000Z', w: '1/1/70' }, - D3: { t: 's', v: '9:02' }, - E3: { t: 'n', w: '-57', v: -57 }, - F3: { t: 'n', w: '-122', v: -122 }, - H3: { t: 's', v: 'North' }, - A4: { t: 'n', w: '180713', v: 180713 }, - B4: { t: 'n', w: '3', v: 3 }, - C4: { z: 'm/d/yy', t: 'd', v: '1970-01-01T08:00:00.000Z', w: '1/1/70' }, - D4: { t: 's', v: '9:03' }, - E4: { t: 'n', w: '-56', v: -56 }, - F4: { t: 'n', w: '-121', v: -121 }, - H4: { t: 's', v: 'North' }, - '!ref': 'A1:H9' - }; - - const result = standard_column_utils.getNonStandardColumnNamesFromWorksheet(xlsxWorksheet); - - expect(result).to.eql(['ANTLER CONFIGURATION', 'WIND DIRECTION']); - }); - }); - - describe('getTsnFromRow', () => { - it('returns the tsn', () => { - const row: Record = { - OTHER: 'other', - ITIS_TSN: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTsnFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the tsn', () => { - const row: Record = { - OTHER: 'other', - TSN: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTsnFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the tsn', () => { - const row: Record = { - OTHER: 'other', - TAXON: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTsnFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the tsn', () => { - const row: Record = { - OTHER: 'other', - SPECIES: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTsnFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns undefined when no known tsn field is present', () => { - const row: Record = { - OTHER: 'other', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTsnFromRow(row); - - expect(result).to.equal(undefined); - }); - }); - - describe('getCountFromRow', () => { - it('returns the count', () => { - const row: Record = { - OTHER: 'other', - COUNT: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getCountFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns undefined when no known count field is present', () => { - const row: Record = { - OTHER: 'other', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getCountFromRow(row); - - expect(result).to.equal(undefined); - }); - }); - - describe('getDateFromRow', () => { - it('returns the date', () => { - const row: Record = { - OTHER: 'other', - DATE: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getDateFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns undefined when no known date field is present', () => { - const row: Record = { - OTHER: 'other', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getDateFromRow(row); - - expect(result).to.equal(undefined); - }); - }); - - describe('getTimeFromRow', () => { - it('returns the time', () => { - const row: Record = { - OTHER: 'other', - TIME: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTimeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns undefined when no known time field is present', () => { - const row: Record = { - OTHER: 'other', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getTimeFromRow(row); - - expect(result).to.equal(undefined); - }); - }); - - describe('getLatitudeFromRow', () => { - it('returns the latitude', () => { - const row: Record = { - OTHER: 'other', - LATITUDE: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLatitudeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the latitude', () => { - const row: Record = { - OTHER: 'other', - LAT: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLatitudeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns undefined when no known latitude field is present', () => { - const row: Record = { - OTHER: 'other', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLatitudeFromRow(row); - - expect(result).to.equal(undefined); - }); - }); - - describe('getLongitudeFromRow', () => { - it('returns the longitude', () => { - const row: Record = { - OTHER: 'other', - LONGITUDE: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLongitudeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the longitude', () => { - const row: Record = { - OTHER: 'other', - LONG: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLongitudeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the longitude', () => { - const row: Record = { - OTHER: 'other', - LON: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLongitudeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns the longitude', () => { - const row: Record = { - OTHER: 'other', - LNG: '123456', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLongitudeFromRow(row); - - expect(result).to.equal('123456'); - }); - - it('returns undefined when no known longitude field is present', () => { - const row: Record = { - OTHER: 'other', - OTHER2: 'other2' - }; - - const result = standard_column_utils.getLongitudeFromRow(row); - - expect(result).to.equal(undefined); - }); - }); -}); diff --git a/api/src/utils/observation-xlsx-utils/standard-column-utils.ts b/api/src/utils/observation-xlsx-utils/standard-column-utils.ts deleted file mode 100644 index eee8745960..0000000000 --- a/api/src/utils/observation-xlsx-utils/standard-column-utils.ts +++ /dev/null @@ -1,134 +0,0 @@ -import xlsx from 'xlsx'; -import { getHeadersUpperCase, IXLSXCSVValidator } from '../xlsx-utils/worksheet-utils'; - -// Observation CSV standard column names and aliases -const ITIS_TSN = 'ITIS_TSN'; -const TAXON = 'TAXON'; -const SPECIES = 'SPECIES'; -const TSN = 'TSN'; - -const COUNT = 'COUNT'; - -const DATE = 'DATE'; - -const TIME = 'TIME'; - -const LATITUDE = 'LATITUDE'; -const LAT = 'LAT'; - -const LONGITUDE = 'LONGITUDE'; -const LON = 'LON'; -const LONG = 'LONG'; -const LNG = 'LNG'; - -/** - * An XLSX validation config for the standard columns of an observation CSV. - */ -export const observationStandardColumnValidator: IXLSXCSVValidator = { - columnNames: [ITIS_TSN, COUNT, DATE, TIME, LATITUDE, LONGITUDE], - columnTypes: ['number', 'number', 'date', 'string', 'number', 'number'], - columnAliases: { - ITIS_TSN: [TAXON, SPECIES, TSN], - LATITUDE: [LAT], - LONGITUDE: [LON, LONG, LNG] - } -}; - -/** - * This function pulls out any non-standard columns from a CSV so they can be processed separately. - * - * @param {xlsx.WorkSheet} xlsxWorksheets The worksheet to pull the columns from - * @returns {*} string[] The list of non-standard columns found in the CSV - */ -export function getNonStandardColumnNamesFromWorksheet(xlsxWorksheet: xlsx.WorkSheet): string[] { - const columns = getHeadersUpperCase(xlsxWorksheet); - - let aliasColumns: string[] = []; - // Create a list of all column names and aliases - if (observationStandardColumnValidator.columnAliases) { - aliasColumns = Object.values(observationStandardColumnValidator.columnAliases).flat(); - } - - const standardColumNames = [...observationStandardColumnValidator.columnNames, ...aliasColumns]; - - // Only return column names not in the validation CSV Column validator (ie: only return the non-standard columns) - return columns.filter((column) => !standardColumNames.includes(column)); -} - -/** - * Get the TSN cell value for a given row. - * - * Note: Requires the row headers to be UPPERCASE. - * - * @export - * @param {Record} row - * @return {*} - */ -export function getTsnFromRow(row: Record) { - return row[ITIS_TSN] ?? row[TSN] ?? row[TAXON] ?? row[SPECIES]; -} - -/** - * Get the count cell value for a given row. - * - * Note: Requires the row headers to be UPPERCASE. - * - * @export - * @param {Record} row - * @return {*} - */ -export function getCountFromRow(row: Record) { - return row[COUNT]; -} - -/** - * Get the date cell value for a given row. - * - * Note: Requires the row headers to be UPPERCASE. - * - * @export - * @param {Record} row - * @return {*} - */ -export function getDateFromRow(row: Record) { - return row[DATE]; -} - -/** - * Get the time cell value for a given row. - * - * Note: Requires the row headers to be UPPERCASE. - * - * @export - * @param {Record} row - * @return {*} - */ -export function getTimeFromRow(row: Record) { - return row[TIME]; -} - -/** - * Get the latitude cell value for a given row. - * - * Note: Requires the row headers to be UPPERCASE. - * - * @export - * @param {Record} row - * @return {*} - */ -export function getLatitudeFromRow(row: Record) { - return row[LATITUDE] ?? row[LAT]; -} - -/** - * Get the longitude cell value for a given row. - * - * Note: Requires the row headers to be UPPERCASE. - * - * @export - * @param {Record} row - * @return {*} - */ -export function getLongitudeFromRow(row: Record) { - return row[LONGITUDE] ?? row[LON] ?? row[LONG] ?? row[LNG]; -} diff --git a/api/src/utils/xlsx-utils/column-cell-utils.test.ts b/api/src/utils/xlsx-utils/column-cell-utils.test.ts new file mode 100644 index 0000000000..58afdc0730 --- /dev/null +++ b/api/src/utils/xlsx-utils/column-cell-utils.test.ts @@ -0,0 +1,64 @@ +import { expect } from 'chai'; +import { generateCellValueGetter, getColumnValidatorSpecification } from './column-cell-utils'; +import { IXLSXCSVValidator } from './worksheet-utils'; + +describe('column-validators', () => { + describe('generateCellValueGetter', () => { + it('should return value if property exists in object', () => { + const getValue = generateCellValueGetter('test', 'property'); + const object = { property: true }; + expect(getValue(object)).to.be.true; + }); + + it('should return undefined if property does not exist in object', () => { + const getValue = generateCellValueGetter('test', 'property'); + const object = { bad: true }; + expect(getValue(object)).to.be.undefined; + }); + }); + + describe('getColumnValidatorSpecification', () => { + it('should return specification format', () => { + const columnValidator: IXLSXCSVValidator = { + columnNames: ['TEST', 'COLUMN'], + columnTypes: ['number', 'string'], + columnAliases: { + TEST: ['ALSO TEST'], + COLUMN: ['ALSO COLUMN'] + } + }; + + const spec = getColumnValidatorSpecification(columnValidator); + + expect(spec).to.be.deep.equal([ + { + columnName: 'TEST', + columnType: 'number', + columnAliases: ['ALSO TEST'] + }, + { + columnName: 'COLUMN', + columnType: 'string', + columnAliases: ['ALSO COLUMN'] + } + ]); + }); + + it('should return specification format without aliases if not defined', () => { + const columnValidator: IXLSXCSVValidator = { + columnNames: ['TEST'], + columnTypes: ['number'] + }; + + const spec = getColumnValidatorSpecification(columnValidator); + + expect(spec).to.be.deep.equal([ + { + columnAliases: undefined, + columnName: 'TEST', + columnType: 'number' + } + ]); + }); + }); +}); diff --git a/api/src/utils/xlsx-utils/column-cell-utils.ts b/api/src/utils/xlsx-utils/column-cell-utils.ts new file mode 100644 index 0000000000..b90218609f --- /dev/null +++ b/api/src/utils/xlsx-utils/column-cell-utils.ts @@ -0,0 +1,123 @@ +import { IXLSXCSVValidator } from './worksheet-utils'; + +type Row = Record; + +// Taxon / species aliases +const ITIS_TSN = 'ITIS_TSN'; +const TAXON = 'TAXON'; +const SPECIES = 'SPECIES'; +const TSN = 'TSN'; + +// DateTime +const DATE = 'DATE'; +const TIME = 'TIME'; + +// Latitude and aliases +const LATITUDE = 'LATITUDE'; +const LAT = 'LAT'; + +// Longitude and aliases +const LONGITUDE = 'LONGITUDE'; +const LON = 'LON'; +const LONG = 'LONG'; +const LNG = 'LNG'; + +// Comment aliases +const COMMENT = 'COMMENT'; +const DESCRIPTION = 'DESCRIPTION'; + +// Critter nickname and aliases +const ALIAS = 'ALIAS'; +const NICKNAME = 'NICKNAME'; + +// Critter sex +const SEX = 'SEX'; + +// Critter Wildlife Health ID and aliases +const WLH_ID = 'WLH_ID'; +const WILDLIFE_HEALTH_ID = 'WILDLIFE_HEALTH_ID'; + +// Observation sub-count +const COUNT = 'COUNT'; + +/** + * An XLSX validation config for the standard columns of an observation CSV. + */ +export const observationStandardColumnValidator: IXLSXCSVValidator = { + columnNames: [ITIS_TSN, COUNT, DATE, TIME, LATITUDE, LONGITUDE], + columnTypes: ['number', 'number', 'date', 'string', 'number', 'number'], + columnAliases: { + ITIS_TSN: [TAXON, SPECIES, TSN], + LATITUDE: [LAT], + LONGITUDE: [LON, LONG, LNG] + } +}; + +/** + * An XLSX validation config for the standard columns of a critter CSV. + */ +export const critterStandardColumnValidator: IXLSXCSVValidator = { + columnNames: [ITIS_TSN, SEX, ALIAS, WLH_ID, DESCRIPTION], + columnTypes: ['number', 'string', 'string', 'string', 'string'], + columnAliases: { + ITIS_TSN: [TAXON, SPECIES, TSN], + DESCRIPTION: [COMMENT], + ALIAS: [NICKNAME] + } +}; + +/** + * Get column validator specification as a readable format. Useful for error handling and logging. + * + * @param {IXLSXCSVValidator} columnValidator - Standard column validator + * @returns {*} + */ +export const getColumnValidatorSpecification = (columnValidator: IXLSXCSVValidator) => { + return columnValidator.columnNames.map((column, index) => ({ + columnName: column, + columnType: columnValidator.columnTypes[index], + columnAliases: columnValidator.columnAliases?.[column] + })); +}; + +/** + * Generate a row value getter function from array of allowed column values. + * + * @example const getTsnFromRow = generateCellValueGetter(ITIS_TSN, TSN, TAXON, SPECIES); + * + * @param {...string[]} headers - Column headers + * @returns {(row: Row) => T | undefined} Row value getter function + */ +export const generateCellValueGetter = (...headers: string[]) => { + return (row: Row) => { + for (const header of headers) { + if (row[header]) { + return row[header] as T; + } + } + }; +}; + +/** + * Row cell value getters. Attempt to retrive a cell value from a list of known column headers. + * + */ +export const getTsnFromRow = generateCellValueGetter(ITIS_TSN, TSN, TAXON, SPECIES); + +export const getCountFromRow = generateCellValueGetter(COUNT); + +export const getDateFromRow = generateCellValueGetter(DATE); + +export const getTimeFromRow = generateCellValueGetter(TIME); + +export const getLatitudeFromRow = generateCellValueGetter(LATITUDE, LAT); + +export const getLongitudeFromRow = generateCellValueGetter(LONGITUDE, LONG, LON, LNG); + +export const getDescriptionFromRow = generateCellValueGetter(DESCRIPTION, COMMENT); + +export const getAliasFromRow = generateCellValueGetter(ALIAS, NICKNAME); + +export const getWlhIdFromRow = generateCellValueGetter(WLH_ID, WILDLIFE_HEALTH_ID); + +export const getSexFromRow = generateCellValueGetter(SEX); diff --git a/api/src/utils/xlsx-utils/worksheet-utils.ts b/api/src/utils/xlsx-utils/worksheet-utils.ts index 9bba6a5298..45b68e460b 100644 --- a/api/src/utils/xlsx-utils/worksheet-utils.ts +++ b/api/src/utils/xlsx-utils/worksheet-utils.ts @@ -9,9 +9,23 @@ import { replaceCellDates, trimCellWhitespace } from './cell-utils'; const defaultLog = getLogger('src/utils/xlsx-utils/worksheet-utils'); export interface IXLSXCSVValidator { - columnNames: string[]; - columnTypes: string[]; - columnAliases?: Record; + /** + * Uppercase column headers + * + * @see column-cell-utils.ts + * + */ + columnNames: Uppercase[]; + /** + * Supported column cell types + * + */ + columnTypes: Array<'string' | 'number' | 'date'>; + /** + * Allowed aliases / mappings for column headers. + * + */ + columnAliases?: Record, Uppercase[]>; } /** @@ -321,3 +335,30 @@ export function validateCsvFile(xlsxWorksheet: xlsx.WorkSheet, columnValidator: return true; } + +/** + * This function pulls out any non-standard columns from a CSV so they can be processed separately. + * + * @param {xlsx.WorkSheet} xlsxWorksheet The worksheet to pull the columns from + * @param {IXLSXCSVValidator} columnValidator The column validator + * @returns {*} string[] The list of non-standard columns found in the CSV + */ +export function getNonStandardColumnNamesFromWorksheet( + xlsxWorksheet: xlsx.WorkSheet, + columnValidator: IXLSXCSVValidator +): string[] { + const columns = getHeadersUpperCase(xlsxWorksheet); + + let aliasColumns: string[] = []; + + // Create a list of all column names and aliases + if (columnValidator.columnAliases) { + aliasColumns = Object.values(columnValidator.columnAliases).flat(); + } + + // Combine the column validator headers and all aliases + const standardColumNames = [...columnValidator.columnNames, ...aliasColumns]; + + // Only return column names not in the validation CSV Column validator (ie: only return the non-standard columns) + return columns.filter((column) => !standardColumNames.includes(column)); +} diff --git a/app/src/constants/i18n.ts b/app/src/constants/i18n.ts index c5c224e965..c21e3411ba 100644 --- a/app/src/constants/i18n.ts +++ b/app/src/constants/i18n.ts @@ -270,7 +270,11 @@ export const SurveyAnimalsI18N = { wlhIdHelp: 'An ID used to identify animals in the BC Wildlife Health Program', sexHelp: 'The sex of this critter. Leave as Unknown if unsure.', telemetryDeviceHelp: - 'Devices transmit telemetry data while they are attached to an animal during a deployment. Animals may have multiple devices and deployments, however a single device may not have overlapping deployments.' + 'Devices transmit telemetry data while they are attached to an animal during a deployment. Animals may have multiple devices and deployments, however a single device may not have overlapping deployments.', + // Animal CSV import strings + importRecordsSuccessSnackbarMessage: 'Animals imported successfully.', + importRecordsErrorDialogTitle: 'Error Importing Animal Records', + importRecordsErrorDialogText: 'An error occurred while importing animal records.' } as const; export const FundingSourceI18N = { diff --git a/app/src/features/surveys/animals/list/components/AnimalListToolbar.tsx b/app/src/features/surveys/animals/list/components/AnimalListToolbar.tsx index 68f09c493e..3b31c76861 100644 --- a/app/src/features/surveys/animals/list/components/AnimalListToolbar.tsx +++ b/app/src/features/surveys/animals/list/components/AnimalListToolbar.tsx @@ -1,10 +1,16 @@ -import { mdiDotsVertical, mdiPlus } from '@mdi/js'; +import { mdiDotsVertical, mdiFileDocumentPlusOutline, mdiPlus } from '@mdi/js'; import { Icon } from '@mdi/react'; import Button from '@mui/material/Button'; import IconButton from '@mui/material/IconButton'; import Toolbar from '@mui/material/Toolbar'; import Typography from '@mui/material/Typography'; +import FileUploadDialog from 'components/dialog/FileUploadDialog'; +import { UploadFileStatus } from 'components/file-upload/FileUploadItem'; +import { SurveyAnimalsI18N } from 'constants/i18n'; +import { DialogContext } from 'contexts/dialogContext'; +import { useBiohubApi } from 'hooks/useBioHubApi'; import { useSurveyContext } from 'hooks/useContext'; +import { useContext, useState } from 'react'; import { Link as RouterLink } from 'react-router-dom'; interface IAnimaListToolbarProps { @@ -20,41 +26,87 @@ interface IAnimaListToolbarProps { * @return {*} */ export const AnimalListToolbar = (props: IAnimaListToolbarProps) => { - const { surveyId, projectId } = useSurveyContext(); + const surveyContext = useSurveyContext(); + + const biohubApi = useBiohubApi(); + + const dialogContext = useContext(DialogContext); + + const [openImportDialog, setOpenImportDialog] = useState(false); + + const handleImportAnimals = async (file: File) => { + try { + await biohubApi.survey.importCrittersFromCsv(file, surveyContext.projectId, surveyContext.surveyId); + surveyContext.critterDataLoader.refresh(surveyContext.projectId, surveyContext.surveyId); + } catch (err: any) { + dialogContext.setErrorDialog({ + dialogTitle: SurveyAnimalsI18N.importRecordsErrorDialogTitle, + dialogText: SurveyAnimalsI18N.importRecordsErrorDialogText, + dialogErrorDetails: [err.message], + open: true, + onClose: () => { + dialogContext.setErrorDialog({ open: false }); + }, + onOk: () => { + dialogContext.setErrorDialog({ open: false }); + } + }); + } finally { + setOpenImportDialog(false); + } + }; return ( - - - Animals ‌ - - ({props.animalCount}) - - - - + setOpenImportDialog(false)} + onUpload={handleImportAnimals} + uploadButtonLabel="Import" + FileUploadProps={{ + dropZoneProps: { maxNumFiles: 1, acceptedFileExtensions: '.csv' }, + status: UploadFileStatus.STAGED }} - aria-label="header-settings" - disabled={!props.checkboxSelectedIdsLength} - onClick={props.handleHeaderMenuClick} - title="Bulk Actions"> - - - + /> + + + Animals ‌ + + ({props.animalCount}) + + + + + + + + + ); }; diff --git a/app/src/hooks/api/useSurveyApi.ts b/app/src/hooks/api/useSurveyApi.ts index 3af8055895..3187b31704 100644 --- a/app/src/hooks/api/useSurveyApi.ts +++ b/app/src/hooks/api/useSurveyApi.ts @@ -534,6 +534,29 @@ const useSurveyApi = (axios: AxiosInstance) => { return data; }; + /** + * Bulk upload Critters from CSV. + * + * @async + * @param {File} file - Critters CSV. + * @param {number} projectId + * @param {number} surveyId + * @returns {Promise} + */ + const importCrittersFromCsv = async ( + file: File, + projectId: number, + surveyId: number + ): Promise<{ survey_critter_ids: number[] }> => { + const formData = new FormData(); + + formData.append('media', file); + + const { data } = await axios.post(`/api/project/${projectId}/survey/${surveyId}/critters/import`, formData); + + return data; + }; + return { createSurvey, getSurveyForView, @@ -557,7 +580,8 @@ const useSurveyApi = (axios: AxiosInstance) => { getDeploymentsInSurvey, updateDeployment, getCritterTelemetry, - removeDeployment + removeDeployment, + importCrittersFromCsv }; }; diff --git a/app/src/hooks/api/useTaxonomyApi.test.tsx b/app/src/hooks/api/useTaxonomyApi.test.tsx index 7974050119..c891f5402a 100644 --- a/app/src/hooks/api/useTaxonomyApi.test.tsx +++ b/app/src/hooks/api/useTaxonomyApi.test.tsx @@ -44,13 +44,13 @@ describe('useTaxonomyApi', () => { searchResponse: [ { tsn: '1', - commonNames: ['something'], - scientificName: 'something' + commonNames: ['Something'], + scientificName: 'Something' }, { tsn: '2', - commonNames: ['anything'], - scientificName: 'anything' + commonNames: ['Anything'], + scientificName: 'Anything' } ] }; @@ -76,13 +76,13 @@ describe('useTaxonomyApi', () => { searchResponse: [ { tsn: '3', - commonNames: ['something'], - scientificName: 'something' + commonNames: ['Something'], + scientificName: 'Something' }, { tsn: '4', - commonNames: ['anything'], - scientificName: 'anything' + commonNames: ['Anything'], + scientificName: 'Anything' } ] }; diff --git a/app/src/hooks/api/useTaxonomyApi.ts b/app/src/hooks/api/useTaxonomyApi.ts index 84b05ee5d6..6aa8d92d52 100644 --- a/app/src/hooks/api/useTaxonomyApi.ts +++ b/app/src/hooks/api/useTaxonomyApi.ts @@ -1,5 +1,6 @@ import { useConfigContext } from 'hooks/useContext'; import { IPartialTaxonomy, ITaxonomy } from 'interfaces/useTaxonomyApi.interface'; +import { startCase } from 'lodash-es'; import qs from 'qs'; import useAxios from './useAxios'; @@ -28,7 +29,7 @@ const useTaxonomyApi = () => { } }); - return data.searchResponse; + return parseSearchResponse(data.searchResponse); }; /** @@ -50,7 +51,7 @@ const useTaxonomyApi = () => { return []; } - return data.searchResponse; + return parseSearchResponse(data.searchResponse); } catch (error) { throw new Error('Failed to fetch Taxon records.'); } @@ -62,4 +63,19 @@ const useTaxonomyApi = () => { }; }; +/** + * Parses the taxon search response into start case. + * + * @template T + * @param {T[]} searchResponse - Array of Taxonomy objects + * @returns {T[]} Correctly cased Taxonomy + */ +const parseSearchResponse = (searchResponse: T[]): T[] => { + return searchResponse.map((taxon) => ({ + ...taxon, + commonNames: taxon.commonNames.map((commonName) => startCase(commonName)), + scientificName: startCase(taxon.scientificName) + })); +}; + export default useTaxonomyApi; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..ba33e3cf6c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "biohubbc", + "lockfileVersion": 3, + "requires": true, + "packages": {} +}