SIMSBIOHUB-584 Bulk Animals (#1319)

Bulk import critters with CSV.

api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.test.ts

@@ -0,0 +1,213 @@
+import { expect } from 'chai';
+import sinon from 'sinon';
+import * as db from '../../../../../../database/db';
+import { HTTP400 } from '../../../../../../errors/http-error';
+import { ImportCrittersService } from '../../../../../../services/import-services/import-critters-service';
+import { parseMulterFile } from '../../../../../../utils/media/media-utils';
+import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';
+import { importCsv } from './import';
+
+import * as fileUtils from '../../../../../../utils/file-utils';
+
+describe('importCsv', () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('returns imported critters', async () => {
+    const mockDBConnection = getMockDBConnection({ open: sinon.stub(), commit: sinon.stub(), release: sinon.stub() });
+    const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+    const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]);
+    const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true);
+
+    const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importCsv();
+
+    await requestHandler(mockReq, mockRes, mockNext);
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+
+    expect(mockFileScan).to.have.been.calledOnceWithExactly(mockFile);
+
+    expect(mockGetDBConnection.calledOnce).to.be.true;
+    expect(mockImportCsv).to.have.been.calledOnceWithExactly(1, parseMulterFile(mockFile));
+    expect(mockRes.json).to.have.been.calledOnceWithExactly({ survey_critter_ids: [1, 2] });
+
+    expect(mockDBConnection.commit).to.have.been.calledOnce;
+    expect(mockDBConnection.release).to.have.been.calledOnce;
+  });
+
+  it('should catch error and rollback if no files in request', async () => {
+    const mockDBConnection = getMockDBConnection({
+      open: sinon.stub(),
+      commit: sinon.stub(),
+      release: sinon.stub(),
+      rollback: sinon.stub()
+    });
+    const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+    const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]);
+    const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true);
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importCsv();
+
+    try {
+      await requestHandler(mockReq, mockRes, mockNext);
+      expect.fail();
+    } catch (err: any) {
+      expect(err.message).to.be.equal('Invalid number of files included. Expected 1 CSV file.');
+    }
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+
+    expect(mockFileScan).to.not.have.been.calledOnce;
+
+    expect(mockGetDBConnection.calledOnce).to.be.true;
+    expect(mockImportCsv).to.not.have.been.called;
+    expect(mockRes.json).to.not.have.been.called;
+
+    expect(mockDBConnection.rollback).to.have.been.called;
+    expect(mockDBConnection.commit).to.not.have.been.called;
+    expect(mockDBConnection.release).to.have.been.called;
+  });
+
+  it('should catch error and rollback if more than 1 file in request', async () => {
+    const mockDBConnection = getMockDBConnection({
+      open: sinon.stub(),
+      commit: sinon.stub(),
+      release: sinon.stub(),
+      rollback: sinon.stub()
+    });
+    const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+    const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]);
+    const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true);
+
+    const mockFile = { originalname: 'test.csv', mimetype: 'test.csv', buffer: Buffer.alloc(1) } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile, mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importCsv();
+
+    try {
+      await requestHandler(mockReq, mockRes, mockNext);
+      expect.fail();
+    } catch (err: any) {
+      expect(err.message).to.be.equal('Invalid number of files included. Expected 1 CSV file.');
+    }
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+
+    expect(mockFileScan).to.not.have.been.calledOnce;
+
+    expect(mockGetDBConnection.calledOnce).to.be.true;
+    expect(mockImportCsv).to.not.have.been.called;
+    expect(mockRes.json).to.not.have.been.called;
+
+    expect(mockDBConnection.rollback).to.have.been.called;
+    expect(mockDBConnection.commit).to.not.have.been.called;
+    expect(mockDBConnection.release).to.have.been.called;
+  });
+
+  it('should catch error and rollback if file is not a csv', async () => {
+    const mockDBConnection = getMockDBConnection({
+      open: sinon.stub(),
+      commit: sinon.stub(),
+      release: sinon.stub(),
+      rollback: sinon.stub()
+    });
+    const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+    const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]);
+    const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(true);
+
+    const mockFile = {
+      originalname: 'test.oops',
+      mimetype: 'test.oops',
+      buffer: Buffer.alloc(1)
+    } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importCsv();
+
+    try {
+      await requestHandler(mockReq, mockRes, mockNext);
+      expect.fail();
+    } catch (err: any) {
+      expect(err).to.be.instanceof(HTTP400);
+      expect(err.message).to.be.contains('Invalid file type.');
+    }
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+
+    expect(mockFileScan).to.not.have.been.calledOnce;
+
+    expect(mockGetDBConnection.calledOnce).to.be.true;
+    expect(mockImportCsv).to.not.have.been.called;
+    expect(mockRes.json).to.not.have.been.called;
+
+    expect(mockDBConnection.rollback).to.have.been.called;
+    expect(mockDBConnection.commit).to.not.have.been.called;
+    expect(mockDBConnection.release).to.have.been.called;
+  });
+
+  it('should catch error and rollback if file contains malware', async () => {
+    const mockDBConnection = getMockDBConnection({
+      open: sinon.stub(),
+      commit: sinon.stub(),
+      release: sinon.stub(),
+      rollback: sinon.stub()
+    });
+    const mockGetDBConnection = sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+    const mockImportCsv = sinon.stub(ImportCrittersService.prototype, 'import').resolves([1, 2]);
+
+    const mockFileScan = sinon.stub(fileUtils, 'scanFileForVirus').resolves(false);
+
+    const mockFile = {
+      originalname: 'test.csv',
+      mimetype: 'test.csv',
+      buffer: Buffer.alloc(1)
+    } as Express.Multer.File;
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.files = [mockFile];
+    mockReq.params.surveyId = '1';
+
+    const requestHandler = importCsv();
+
+    try {
+      await requestHandler(mockReq, mockRes, mockNext);
+      expect.fail();
+    } catch (err: any) {
+      expect(err).to.be.instanceof(HTTP400);
+      expect(err.message).to.be.contains('Malicious content detected');
+    }
+
+    expect(mockDBConnection.open).to.have.been.calledOnce;
+    expect(mockFileScan).to.have.been.calledOnceWithExactly(mockFile);
+
+    expect(mockGetDBConnection.calledOnce).to.be.true;
+    expect(mockImportCsv).to.not.have.been.called;
+    expect(mockRes.json).to.not.have.been.called;
+
+    expect(mockDBConnection.rollback).to.have.been.called;
+    expect(mockDBConnection.commit).to.not.have.been.called;
+    expect(mockDBConnection.release).to.have.been.called;
+  });
+});

api/src/paths/project/{projectId}/survey/{surveyId}/critters/import.ts

@@ -0,0 +1,169 @@
+import { RequestHandler } from 'express';
+import { Operation } from 'express-openapi';
+import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
+import { getDBConnection } from '../../../../../../database/db';
+import { HTTP400 } from '../../../../../../errors/http-error';
+import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
+import { ImportCrittersService } from '../../../../../../services/import-services/import-critters-service';
+import { scanFileForVirus } from '../../../../../../utils/file-utils';
+import { getLogger } from '../../../../../../utils/logger';
+import { parseMulterFile } from '../../../../../../utils/media/media-utils';
+
+const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/critters/import');
+
+export const POST: Operation = [
+  authorizeRequestHandler((req) => {
+    return {
+      or: [
+        {
+          validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR],
+          surveyId: Number(req.params.surveyId),
+          discriminator: 'ProjectPermission'
+        },
+        {
+          validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR],
+          discriminator: 'SystemRole'
+        }
+      ]
+    };
+  }),
+  importCsv()
+];
+
+POST.apiDoc = {
+  description: 'Upload survey critters submission file',
+  tags: ['critterbase', 'survey'],
+  security: [
+    {
+      Bearer: []
+    }
+  ],
+  parameters: [
+    {
+      in: 'path',
+      name: 'projectId',
+      required: true,
+      schema: {
+        type: 'integer',
+        minimum: 1
+      }
+    },
+    {
+      in: 'path',
+      name: 'surveyId',
+      required: true,
+      schema: {
+        type: 'integer',
+        minimum: 1
+      }
+    }
+  ],
+  requestBody: {
+    description: 'Survey critters submission file to import',
+    content: {
+      'multipart/form-data': {
+        schema: {
+          type: 'object',
+          additionalProperties: false,
+          required: ['media'],
+          properties: {
+            media: {
+              description: 'Survey Critters submission import file.',
+              type: 'string',
+              format: 'binary'
+            }
+          }
+        }
+      }
+    }
+  },
+  responses: {
+    200: {
+      description: 'Import OK',
+      content: {
+        'application/json': {
+          schema: {
+            type: 'object',
+            additionalProperties: false,
+            required: ['survey_critter_ids'],
+            properties: {
+              survey_critter_ids: {
+                type: 'array',
+                items: {
+                  type: 'integer',
+                  minimum: 1
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    400: {
+      $ref: '#/components/responses/400'
+    },
+    401: {
+      $ref: '#/components/responses/401'
+    },
+    403: {
+      $ref: '#/components/responses/403'
+    },
+    500: {
+      $ref: '#/components/responses/500'
+    },
+    default: {
+      $ref: '#/components/responses/default'
+    }
+  }
+};
+
+/**
+ * Imports a `Critter CSV` which adds critters to `survey_critter` table and creates critters in Critterbase.
+ *
+ * @return {*}  {RequestHandler}
+ */
+export function importCsv(): RequestHandler {
+  return async (req, res) => {
+    const surveyId = Number(req.params.surveyId);
+    const rawFiles = req.files as Express.Multer.File[];
+    const rawFile = rawFiles[0];
+
+    const connection = getDBConnection(req['keycloak_token']);
+
+    try {
+      await connection.open();
+
+      if (rawFiles.length !== 1) {
+        throw new HTTP400('Invalid number of files included. Expected 1 CSV file.');
+      }
+
+      if (!rawFile?.originalname.endsWith('.csv')) {
+        throw new HTTP400('Invalid file type. Expected a CSV file.');
+      }
+
+      // Check for viruses / malware
+      const virusScanResult = await scanFileForVirus(rawFile);
+
+      if (!virusScanResult) {
+        throw new HTTP400('Malicious content detected, import cancelled.');
+      }
+
+      const csvImporter = new ImportCrittersService(connection);
+
+      // Pass the survey id and the csv (MediaFile) to the importer
+      const surveyCritterIds = await csvImporter.import(surveyId, parseMulterFile(rawFile));
+
+      defaultLog.info({ label: 'importCritterCsv', message: 'result', survey_critter_ids: surveyCritterIds });
+
+      await connection.commit();
+
+      return res.status(200).json({ survey_critter_ids: surveyCritterIds });
+    } catch (error) {
+      defaultLog.error({ label: 'uploadMedia', message: 'error', error });
+      await connection.rollback();
+      throw error;
+    } finally {
+      connection.release();
+    }
+  };
+}