SIMSBIOHUB-562: Export Survey Data

Makefile

@@ -26,8 +26,7 @@ backend: | close build-backend run-backend ## Performs all commands necessary to
 web: | close build-web check-env run-web ## Performs all commands necessary to run all backend+web projects (db, api, app) in docker
 
 db-setup: | build-db-setup run-db-setup ## Performs all commands necessary to run the database migrations and seeding
-db-migrate: | build-db-migrate run-db-migrate ## Performs all commands necessary to run the database migrations
-db-rollback: | build-db-rollback run-db-rollback ## Performs all commands necessary to rollback the latest database migrations
+
 clamav: | build-clamav run-clamav ## Performs all commands necessary to run clamav
 
 fix: | lint-fix format-fix ## Performs both lint-fix and format-fix commands
@@ -158,30 +157,6 @@ run-db-setup: ## Run the database migrations and seeding
 	@echo "==============================================="
 	@docker compose up db_setup
 
-build-db-migrate: ## Build the db knex migrations image
-	@echo "==============================================="
-	@echo "Make: build-db-migrate - building db knex migrate image"
-	@echo "==============================================="
-	@docker compose build db_migrate
-
-run-db-migrate: ## Run the database migrations
-	@echo "==============================================="
-	@echo "Make: run-db-migrate - running database migrations"
-	@echo "==============================================="
-	@docker compose up db_migrate
-
-build-db-rollback: ## Build the db knex rollback image
-	@echo "==============================================="
-	@echo "Make: build-db-rollback - building db knex rollback image"
-	@echo "==============================================="
-	@docker compose build db_rollback
-
-run-db-rollback: ## Rollback the latest database migrations
-	@echo "==============================================="
-	@echo "Make: run-db-rollback - rolling back the latest database migrations"
-	@echo "==============================================="
-	@docker compose up db_rollback
-
 ## ------------------------------------------------------------------------------
 ## clamav commands
 ## ------------------------------------------------------------------------------

api/package.json

@@ -27,13 +27,15 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.583.0",
+    "@aws-sdk/lib-storage": "^3.621.0",
     "@aws-sdk/s3-request-presigner": "^3.583.0",
     "@turf/bbox": "^6.5.0",
     "@turf/circle": "^6.5.0",
     "@turf/helpers": "^6.5.0",
     "@turf/meta": "^6.5.0",
     "adm-zip": "0.5.12",
     "ajv": "^8.12.0",
+    "archiver": "^7.0.1",
     "axios": "^1.6.7",
     "clamscan": "^2.2.1",
     "dayjs": "^1.11.10",
@@ -55,6 +57,7 @@
     "mime": "^3.0.0",
     "multer": "^1.4.5-lts.1",
     "pg": "^8.7.1",
+    "pg-query-stream": "^4.5.5",
     "qs": "^6.10.1",
     "sql-template-strings": "^2.2.2",
     "swagger-ui-express": "^4.3.0",
@@ -69,6 +72,7 @@
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.1",
     "@types/adm-zip": "^0.4.34",
+    "@types/archiver": "^6.0.2",
     "@types/chai": "^4.3.14",
     "@types/clamscan": "^2.0.8",
     "@types/express": "^4.17.13",

api/src/__mocks__/db.ts

@@ -1,5 +1,5 @@
 import { Request, Response } from 'express';
-import { QueryResult } from 'pg';
+import { PoolClient, QueryResult } from 'pg';
 import sinon from 'sinon';
 import * as db from '../database/db';
 import { IDBConnection } from '../database/db';
@@ -35,6 +35,9 @@
     systemUserIdentifier: () => {
       return null as unknown as string;
     },
+    getClient: async () => {
+      return null as unknown as PoolClient;
+    },
     open: async () => {
       // do nothing
     },
@@ -47,9 +50,6 @@
     rollback: async () => {
       // do nothing
     },
-    query: async () => {
-      return undefined as unknown as QueryResult<any>;
-    },
     sql: async () => {
       return undefined as unknown as QueryResult<any>;
     },

api/src/database/db.test.ts

@@ -266,52 +266,6 @@ describe('db', () => {
         });
       });
 
-      describe('query', () => {
-        describe('when a connection is open', () => {
-          it('sends a query with values', async () => {
-            sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);
-
-            await connection.open();
-
-            await connection.query('sql query', ['value1', 'value2']);
-
-            expect(queryStub).to.have.been.calledWith('sql query', ['value1', 'value2']);
-          });
-
-          it('sends a query with empty values', async () => {
-            sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);
-
-            await connection.open();
-
-            await connection.query('sql query');
-
-            expect(queryStub).to.have.been.calledWith('sql query', []);
-          });
-        });
-
-        describe('when a connection is not open', () => {
-          it('throws an error', async () => {
-            sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);
-
-            let expectedError: ApiExecuteSQLError;
-            try {
-              await connection.query('sql query');
-
-              expect.fail('Expected an error to be thrown');
-            } catch (error) {
-              expectedError = error as ApiExecuteSQLError;
-            }
-
-            expect(expectedError.message).to.equal('Failed to execute SQL');
-
-            expect(expectedError.errors?.length).to.be.greaterThan(0);
-            expectedError.errors?.forEach((item) => {
-              expect(item).to.be.eql({ name: 'Error', message: 'DBConnection is not open' });
-            });
-          });
-        });
-      });
-
       describe('sql', () => {
         describe('when a connection is open', () => {
           it('sends a sql statement', async () => {

api/src/database/db.ts

@@ -102,6 +102,15 @@
 };
 
 export interface IDBConnection {
+  /**
+   * Get a new pg client.
+   *
+   * Note: This is not the same client that is initialized when calling `.open()`, and must be released manually by
+   * calling `client.release()`.
+   *
+   * @memberof IDBConnection
+   */
+  getClient: () => Promise<pg.PoolClient>;
   /**
    * Opens a new connection, begins a transaction, and sets the user context.
    *
@@ -132,17 +141,6 @@
    * @memberof IDBConnection
    */
   rollback: () => Promise<void>;
-  /**
-   * Performs a query against this connection, returning the results.
-   *
-   * @param {string} text SQL text
-   * @param {any[]} [values] SQL values array (optional)
-   * @return {*}  {(Promise<QueryResult<any>>)}
-   * @throws If the connection is not open.
-   * @deprecated Prefer using `.sql` (pass entire statement object) or `.knex` (pass knex query builder object)
-   * @memberof IDBConnection
-   */
-  query: <T extends pg.QueryResultRow = any>(text: string, values?: any[]) => Promise<pg.QueryResult<T>>;
   /**
    * Performs a query against this connection, returning the results.
    *
@@ -236,6 +234,21 @@
   let _systemUserId: number | null = null;
   const _token = keycloakToken;
 
+  /**
+   * Get a new pg client.
+   *
+   * @return {*}
+   */
+  const _getClient = async () => {
+    const pool = getDBPool();
+
+    if (!pool) {
+      throw Error('DBPool is not initialized');
+    }
+
+    return pool.connect();
+  };
+
   /**
    * Opens a new connection, begins a transaction, and sets the user context.
    *
@@ -553,8 +566,8 @@
   };
 
   return {
+    getClient: asyncErrorWrapper(_getClient),
     open: asyncErrorWrapper(_open),
-    query: asyncErrorWrapper(_query),
     sql: asyncErrorWrapper(_sql),
     knex: asyncErrorWrapper(_knex),
     release: syncErrorWrapper(_release),

api/src/models/animal-view.ts

@@ -1,6 +1,37 @@
 export interface IAnimalAdvancedFilters {
+  /**
+   * Filter results by keyword.
+   *
+   * @type {string}
+   * @memberof IAnimalAdvancedFilters
+   */
   keyword?: string;
+  /**
+   * Filter results by ITIS TSNs.
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsns?: number[];
+  /**
+   * Filter results by ITIS TSN.
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsn?: number;
+  /**
+   * Filter results by system user id (not necessarily the user making the request).
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   system_user_id?: number;
+  /**
+   * Filter results by survey ids
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
+  survey_ids?: number[];
 }

api/src/models/telemetry-view.ts

@@ -1,6 +1,37 @@
 export interface IAllTelemetryAdvancedFilters {
+  /**
+   * Filter results by keyword.
+   *
+   * @type {string}
+   * @memberof IAnimalAdvancedFilters
+   */
   keyword?: string;
+  /**
+   * Filter results by ITIS TSNs.
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsns?: number[];
+  /**
+   * Filter results by ITIS TSN.
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   itis_tsn?: number;
+  /**
+   * Filter results by system user id (not necessarily the user making the request).
+   *
+   * @type {number}
+   * @memberof IAnimalAdvancedFilters
+   */
   system_user_id?: number;
+  /**
+   * Filter results by survey ids
+   *
+   * @type {number[]}
+   * @memberof IAnimalAdvancedFilters
+   */
+  survey_ids?: number[];
 }

api/src/paths/project/{projectId}/survey/{surveyId}/export/index.test.ts

@@ -0,0 +1,98 @@
+import chai, { expect } from 'chai';
+import { describe } from 'mocha';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import { exportData } from '.';
+import { SYSTEM_ROLE } from '../../../../../../constants/roles';
+import * as db from '../../../../../../database/db';
+import { HTTPError } from '../../../../../../errors/http-error';
+import { SystemUser } from '../../../../../../repositories/user-repository';
+import { ExportService } from '../../../../../../services/export-services/export-service';
+import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';
+
+chai.use(sinonChai);
+
+describe('exportData', () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('catches and re-throws error', async () => {
+    // Setup
+    const mockDBConnection = getMockDBConnection({ release: sinon.stub() });
+    sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+
+    sinon.stub(ExportService.prototype, 'export').rejects(new Error('a test error'));
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.system_user = {
+      system_user_id: 1,
+      role_names: [SYSTEM_ROLE.PROJECT_CREATOR]
+    } as SystemUser;
+
+    mockReq.params = {
+      projectId: '1',
+      surveyId: '2'
+    };
+
+    mockReq.body = {
+      methodTechniqueIds: [1, 2, 3]
+    };
+
+    const requestHandler = exportData();
+
+    try {
+      // Execute
+      await requestHandler(mockReq, mockRes, mockNext);
+
+      expect.fail('Expected an error to be thrown');
+    } catch (actualError) {
+      // Assert
+      expect(mockDBConnection.release).to.have.been.calledOnce;
+
+      expect((actualError as HTTPError).message).to.equal('a test error');
+    }
+  });
+
+  it('returns the s3 signed url for the export data file', async () => {
+    // Setup
+    const mockDBConnection = getMockDBConnection({ release: sinon.stub() });
+    sinon.stub(db, 'getDBConnection').returns(mockDBConnection);
+
+    sinon.stub(ExportService.prototype, 'export').resolves(['signed-url-for:path/to/file/key']);
+
+    const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
+
+    mockReq.system_user = {
+      system_user_id: 1,
+      role_names: [SYSTEM_ROLE.PROJECT_CREATOR]
+    } as SystemUser;
+
+    mockReq.params = {
+      projectId: '1',
+      surveyId: '2'
+    };
+
+    mockReq.body = {
+      config: {
+        metadata: true,
+        sampling_data: false,
+        observation_data: true,
+        telemetry_data: true,
+        animal_data: false,
+        artifacts: false
+      }
+    };
+
+    // Execute
+    const requestHandler = exportData();
+
+    await requestHandler(mockReq, mockRes, mockNext);
+
+    // Assert
+    expect(mockRes.jsonValue).to.eql({ presignedS3Urls: ['signed-url-for:path/to/file/key'] });
+
+    expect(mockDBConnection.release).to.have.been.calledOnce;
+  });
+});