Skip to content

Commit

Permalink
SIMSBIOHUB-562: Export Survey Data (#1273)
Browse files Browse the repository at this point in the history
* Initial commit

* Updates

* Initial survey export UIs

* Copy changes from biohub

* Experimental stream

* Updates

* Updates

* Update export backend

* Fix unit test

* ignore-skip

* Tweaks

* Updates to support export stream configs

* Update config

* Working with api strams

* Revert virus scan change

* ignore-skip

* styling for export dialog

* Remove debug log

* Tweaks

* Fix error message

* Update error logging, add unit tests.

* Add more unit tests

---------

Co-authored-by: Macgregor Aubertin-Young <[email protected]>
Co-authored-by: Macgregor Aubertin-Young <[email protected]>
  • Loading branch information
3 people authored Sep 17, 2024
1 parent 556ba8d commit 1c26829
Show file tree
Hide file tree
Showing 37 changed files with 3,776 additions and 1,016 deletions.
27 changes: 1 addition & 26 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ backend: | close build-backend run-backend ## Performs all commands necessary to
web: | close build-web check-env run-web ## Performs all commands necessary to run all backend+web projects (db, api, app) in docker

db-setup: | build-db-setup run-db-setup ## Performs all commands necessary to run the database migrations and seeding
db-migrate: | build-db-migrate run-db-migrate ## Performs all commands necessary to run the database migrations
db-rollback: | build-db-rollback run-db-rollback ## Performs all commands necessary to rollback the latest database migrations

clamav: | build-clamav run-clamav ## Performs all commands necessary to run clamav

fix: | lint-fix format-fix ## Performs both lint-fix and format-fix commands
Expand Down Expand Up @@ -158,30 +157,6 @@ run-db-setup: ## Run the database migrations and seeding
@echo "==============================================="
@docker compose up db_setup

build-db-migrate: ## Build the db knex migrations image
@echo "==============================================="
@echo "Make: build-db-migrate - building db knex migrate image"
@echo "==============================================="
@docker compose build db_migrate

run-db-migrate: ## Run the database migrations
@echo "==============================================="
@echo "Make: run-db-migrate - running database migrations"
@echo "==============================================="
@docker compose up db_migrate

build-db-rollback: ## Build the db knex rollback image
@echo "==============================================="
@echo "Make: build-db-rollback - building db knex rollback image"
@echo "==============================================="
@docker compose build db_rollback

run-db-rollback: ## Rollback the latest database migrations
@echo "==============================================="
@echo "Make: run-db-rollback - rolling back the latest database migrations"
@echo "==============================================="
@docker compose up db_rollback

## ------------------------------------------------------------------------------
## clamav commands
## ------------------------------------------------------------------------------
Expand Down
2,498 changes: 1,784 additions & 714 deletions api/package-lock.json

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,15 @@
},
"dependencies": {
"@aws-sdk/client-s3": "^3.583.0",
"@aws-sdk/lib-storage": "^3.621.0",
"@aws-sdk/s3-request-presigner": "^3.583.0",
"@turf/bbox": "^6.5.0",
"@turf/circle": "^6.5.0",
"@turf/helpers": "^6.5.0",
"@turf/meta": "^6.5.0",
"adm-zip": "0.5.12",
"ajv": "^8.12.0",
"archiver": "^7.0.1",
"axios": "^1.6.7",
"clamscan": "^2.2.1",
"dayjs": "^1.11.10",
Expand All @@ -55,6 +57,7 @@
"mime": "^3.0.0",
"multer": "^1.4.5-lts.1",
"pg": "^8.7.1",
"pg-query-stream": "^4.5.5",
"qs": "^6.10.1",
"sql-template-strings": "^2.2.2",
"swagger-ui-express": "^4.3.0",
Expand All @@ -69,6 +72,7 @@
"devDependencies": {
"@istanbuljs/nyc-config-typescript": "^1.0.1",
"@types/adm-zip": "^0.4.34",
"@types/archiver": "^6.0.2",
"@types/chai": "^4.3.14",
"@types/clamscan": "^2.0.8",
"@types/express": "^4.17.13",
Expand Down
8 changes: 4 additions & 4 deletions api/src/__mocks__/db.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Request, Response } from 'express';
import { QueryResult } from 'pg';
import { PoolClient, QueryResult } from 'pg';
import sinon from 'sinon';
import * as db from '../database/db';
import { IDBConnection } from '../database/db';
Expand Down Expand Up @@ -35,6 +35,9 @@ export const getMockDBConnection = (config?: Partial<IDBConnection>): IDBConnect
systemUserIdentifier: () => {
return null as unknown as string;
},
getClient: async () => {
return null as unknown as PoolClient;

Check warning on line 39 in api/src/__mocks__/db.ts

View check run for this annotation

Codecov / codecov/patch

api/src/__mocks__/db.ts#L39

Added line #L39 was not covered by tests
},
open: async () => {
// do nothing
},
Expand All @@ -47,9 +50,6 @@ export const getMockDBConnection = (config?: Partial<IDBConnection>): IDBConnect
rollback: async () => {
// do nothing
},
query: async () => {
return undefined as unknown as QueryResult<any>;
},
sql: async () => {
return undefined as unknown as QueryResult<any>;
},
Expand Down
46 changes: 0 additions & 46 deletions api/src/database/db.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -266,52 +266,6 @@ describe('db', () => {
});
});

describe('query', () => {
describe('when a connection is open', () => {
it('sends a query with values', async () => {
sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);

await connection.open();

await connection.query('sql query', ['value1', 'value2']);

expect(queryStub).to.have.been.calledWith('sql query', ['value1', 'value2']);
});

it('sends a query with empty values', async () => {
sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);

await connection.open();

await connection.query('sql query');

expect(queryStub).to.have.been.calledWith('sql query', []);
});
});

describe('when a connection is not open', () => {
it('throws an error', async () => {
sinonSandbox.stub(db, 'getDBPool').returns(mockPool as unknown as pg.Pool);

let expectedError: ApiExecuteSQLError;
try {
await connection.query('sql query');

expect.fail('Expected an error to be thrown');
} catch (error) {
expectedError = error as ApiExecuteSQLError;
}

expect(expectedError.message).to.equal('Failed to execute SQL');

expect(expectedError.errors?.length).to.be.greaterThan(0);
expectedError.errors?.forEach((item) => {
expect(item).to.be.eql({ name: 'Error', message: 'DBConnection is not open' });
});
});
});
});

describe('sql', () => {
describe('when a connection is open', () => {
it('sends a sql statement', async () => {
Expand Down
37 changes: 25 additions & 12 deletions api/src/database/db.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ export const getDBPool = function (): pg.Pool | undefined {
};

export interface IDBConnection {
/**
* Get a new pg client.
*
* Note: This is not the same client that is initialized when calling `.open()`, and must be released manually by
* calling `client.release()`.
*
* @memberof IDBConnection
*/
getClient: () => Promise<pg.PoolClient>;
/**
* Opens a new connection, begins a transaction, and sets the user context.
*
Expand Down Expand Up @@ -132,17 +141,6 @@ export interface IDBConnection {
* @memberof IDBConnection
*/
rollback: () => Promise<void>;
/**
* Performs a query against this connection, returning the results.
*
* @param {string} text SQL text
* @param {any[]} [values] SQL values array (optional)
* @return {*} {(Promise<QueryResult<any>>)}
* @throws If the connection is not open.
* @deprecated Prefer using `.sql` (pass entire statement object) or `.knex` (pass knex query builder object)
* @memberof IDBConnection
*/
query: <T extends pg.QueryResultRow = any>(text: string, values?: any[]) => Promise<pg.QueryResult<T>>;
/**
* Performs a query against this connection, returning the results.
*
Expand Down Expand Up @@ -236,6 +234,21 @@ export const getDBConnection = function (keycloakToken?: KeycloakUserInformation
let _systemUserId: number | null = null;
const _token = keycloakToken;

/**
* Get a new pg client.
*
* @return {*}
*/
const _getClient = async () => {
const pool = getDBPool();

Check warning on line 243 in api/src/database/db.ts

View check run for this annotation

Codecov / codecov/patch

api/src/database/db.ts#L243

Added line #L243 was not covered by tests

if (!pool) {
throw Error('DBPool is not initialized');

Check warning on line 246 in api/src/database/db.ts

View check run for this annotation

Codecov / codecov/patch

api/src/database/db.ts#L246

Added line #L246 was not covered by tests
}

return pool.connect();

Check warning on line 249 in api/src/database/db.ts

View check run for this annotation

Codecov / codecov/patch

api/src/database/db.ts#L249

Added line #L249 was not covered by tests
};

/**
* Opens a new connection, begins a transaction, and sets the user context.
*
Expand Down Expand Up @@ -553,8 +566,8 @@ export const getDBConnection = function (keycloakToken?: KeycloakUserInformation
};

return {
getClient: asyncErrorWrapper(_getClient),
open: asyncErrorWrapper(_open),
query: asyncErrorWrapper(_query),
sql: asyncErrorWrapper(_sql),
knex: asyncErrorWrapper(_knex),
release: syncErrorWrapper(_release),
Expand Down
31 changes: 31 additions & 0 deletions api/src/models/animal-view.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,37 @@
export interface IAnimalAdvancedFilters {
/**
* Filter results by keyword.
*
* @type {string}
* @memberof IAnimalAdvancedFilters
*/
keyword?: string;
/**
* Filter results by ITIS TSNs.
*
* @type {number[]}
* @memberof IAnimalAdvancedFilters
*/
itis_tsns?: number[];
/**
* Filter results by ITIS TSN.
*
* @type {number}
* @memberof IAnimalAdvancedFilters
*/
itis_tsn?: number;
/**
* Filter results by system user id (not necessarily the user making the request).
*
* @type {number}
* @memberof IAnimalAdvancedFilters
*/
system_user_id?: number;
/**
* Filter results by survey ids
*
* @type {number[]}
* @memberof IAnimalAdvancedFilters
*/
survey_ids?: number[];
}
31 changes: 31 additions & 0 deletions api/src/models/telemetry-view.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,37 @@
export interface IAllTelemetryAdvancedFilters {
/**
* Filter results by keyword.
*
* @type {string}
* @memberof IAnimalAdvancedFilters
*/
keyword?: string;
/**
* Filter results by ITIS TSNs.
*
* @type {number[]}
* @memberof IAnimalAdvancedFilters
*/
itis_tsns?: number[];
/**
* Filter results by ITIS TSN.
*
* @type {number}
* @memberof IAnimalAdvancedFilters
*/
itis_tsn?: number;
/**
* Filter results by system user id (not necessarily the user making the request).
*
* @type {number}
* @memberof IAnimalAdvancedFilters
*/
system_user_id?: number;
/**
* Filter results by survey ids
*
* @type {number[]}
* @memberof IAnimalAdvancedFilters
*/
survey_ids?: number[];
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import chai, { expect } from 'chai';
import { describe } from 'mocha';
import sinon from 'sinon';
import sinonChai from 'sinon-chai';
import { exportData } from '.';
import { SYSTEM_ROLE } from '../../../../../../constants/roles';
import * as db from '../../../../../../database/db';
import { HTTPError } from '../../../../../../errors/http-error';
import { SystemUser } from '../../../../../../repositories/user-repository';
import { ExportService } from '../../../../../../services/export-services/export-service';
import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';

chai.use(sinonChai);

describe('exportData', () => {
afterEach(() => {
sinon.restore();
});

it('catches and re-throws error', async () => {
// Setup
const mockDBConnection = getMockDBConnection({ release: sinon.stub() });
sinon.stub(db, 'getDBConnection').returns(mockDBConnection);

sinon.stub(ExportService.prototype, 'export').rejects(new Error('a test error'));

const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();

mockReq.system_user = {
system_user_id: 1,
role_names: [SYSTEM_ROLE.PROJECT_CREATOR]
} as SystemUser;

mockReq.params = {
projectId: '1',
surveyId: '2'
};

mockReq.body = {
methodTechniqueIds: [1, 2, 3]
};

const requestHandler = exportData();

try {
// Execute
await requestHandler(mockReq, mockRes, mockNext);

expect.fail('Expected an error to be thrown');
} catch (actualError) {
// Assert
expect(mockDBConnection.release).to.have.been.calledOnce;

expect((actualError as HTTPError).message).to.equal('a test error');
}
});

it('returns the s3 signed url for the export data file', async () => {
// Setup
const mockDBConnection = getMockDBConnection({ release: sinon.stub() });
sinon.stub(db, 'getDBConnection').returns(mockDBConnection);

sinon.stub(ExportService.prototype, 'export').resolves(['signed-url-for:path/to/file/key']);

const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();

mockReq.system_user = {
system_user_id: 1,
role_names: [SYSTEM_ROLE.PROJECT_CREATOR]
} as SystemUser;

mockReq.params = {
projectId: '1',
surveyId: '2'
};

mockReq.body = {
config: {
metadata: true,
sampling_data: false,
observation_data: true,
telemetry_data: true,
animal_data: false,
artifacts: false
}
};

// Execute
const requestHandler = exportData();

await requestHandler(mockReq, mockRes, mockNext);

// Assert
expect(mockRes.jsonValue).to.eql({ presignedS3Urls: ['signed-url-for:path/to/file/key'] });

expect(mockDBConnection.release).to.have.been.calledOnce;
});
});
Loading

0 comments on commit 1c26829

Please sign in to comment.