Skip to content

Commit

Permalink
SIMSBIOHUB-426-2: Calculate and display submission regions (#232)
Browse files Browse the repository at this point in the history
SIMSBIOHUB-426: Added support for calculating regions for a submission
  • Loading branch information
al-rosenthal authored Jan 17, 2024
1 parent a305328 commit 9084dec
Show file tree
Hide file tree
Showing 28 changed files with 645 additions and 211 deletions.
6 changes: 4 additions & 2 deletions api/src/paths/administrative/submission/published.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ describe('getPublishedSubmissionsForAdmins', () => {
revision_count: 0,
security: SECURITY_APPLIED_STATUS.SECURED,
root_feature_type_id: 1,
root_feature_type_name: 'dataset'
root_feature_type_name: 'dataset',
regions: []
},
{
submission_id: 2,
Expand All @@ -82,7 +83,8 @@ describe('getPublishedSubmissionsForAdmins', () => {
revision_count: 1,
security: SECURITY_APPLIED_STATUS.PARTIALLY_SECURED,
root_feature_type_id: 1,
root_feature_type_name: 'dataset'
root_feature_type_name: 'dataset',
regions: []
}
];

Expand Down
6 changes: 4 additions & 2 deletions api/src/paths/administrative/submission/reviewed.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ describe('getReviewedSubmissionsForAdmins', () => {
revision_count: 0,
security: SECURITY_APPLIED_STATUS.SECURED,
root_feature_type_id: 1,
root_feature_type_name: 'dataset'
root_feature_type_name: 'dataset',
regions: []
},
{
submission_id: 2,
Expand All @@ -82,7 +83,8 @@ describe('getReviewedSubmissionsForAdmins', () => {
revision_count: 1,
security: SECURITY_APPLIED_STATUS.PARTIALLY_SECURED,
root_feature_type_id: 1,
root_feature_type_name: 'dataset'
root_feature_type_name: 'dataset',
regions: []
}
];

Expand Down
6 changes: 4 additions & 2 deletions api/src/paths/administrative/submission/unreviewed.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ describe('getUnreviewedSubmissionsForAdmins', () => {
revision_count: 0,
security: SECURITY_APPLIED_STATUS.PENDING,
root_feature_type_id: 1,
root_feature_type_name: 'dataset'
root_feature_type_name: 'dataset',
regions: []
},
{
submission_id: 2,
Expand All @@ -82,7 +83,8 @@ describe('getUnreviewedSubmissionsForAdmins', () => {
revision_count: 1,
security: SECURITY_APPLIED_STATUS.PENDING,
root_feature_type_id: 1,
root_feature_type_name: 'dataset'
root_feature_type_name: 'dataset',
regions: []
}
];

Expand Down
9 changes: 8 additions & 1 deletion api/src/paths/administrative/submission/unreviewed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ GET.apiDoc = {
'revision_count',
'security',
'root_feature_type_id',
'root_feature_type_name'
'root_feature_type_name',
'regions'
],
properties: {
submission_id: {
Expand Down Expand Up @@ -115,6 +116,12 @@ GET.apiDoc = {
},
root_feature_type_name: {
type: 'string'
},
regions: {
type: 'array',
items: {
type: 'string'
}
}
}
}
Expand Down
28 changes: 0 additions & 28 deletions api/src/paths/dwc/submission/{datasetId}/handlebar.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ import { describe } from 'mocha';
import sinon from 'sinon';
import sinonChai from 'sinon-chai';
import * as db from '../../../../database/db';
import { HTTPError } from '../../../../errors/http-error';
import { SubmissionService } from '../../../../services/submission-service';
import { getMockDBConnection, getRequestHandlerMocks } from '../../../../__mocks__/db';
import { getHandleBarsTemplateByDatasetId } from './handlebar';

Expand All @@ -20,10 +18,6 @@ describe('handlebar', () => {
const dbConnectionObj = getMockDBConnection();
sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);

sinon
.stub(SubmissionService.prototype, 'getHandleBarsTemplateByDatasetId')
.resolves({ header: 'hedaer', details: 'details' });

const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();

mockReq.params = {
Expand All @@ -36,27 +30,5 @@ describe('handlebar', () => {

expect(mockRes.statusValue).to.equal(200);
});

it('catches and re-throws an error', async () => {
const dbConnectionObj = getMockDBConnection();
sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);

sinon.stub(SubmissionService.prototype, 'getHandleBarsTemplateByDatasetId').rejects(new Error('a test error'));

const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();

mockReq.params = {
datasetId: 'abcd'
};

try {
const requestHandler = getHandleBarsTemplateByDatasetId();

await requestHandler(mockReq, mockRes, mockNext);
expect.fail();
} catch (actualError) {
expect((actualError as HTTPError).message).to.equal('a test error');
}
});
});
});
9 changes: 4 additions & 5 deletions api/src/paths/dwc/submission/{datasetId}/handlebar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { RequestHandler } from 'express';
import { Operation } from 'express-openapi';
import { getAPIUserDBConnection, getDBConnection } from '../../../../database/db';
import { defaultErrorResponses } from '../../../../openapi/schemas/http-responses';
import { SubmissionService } from '../../../../services/submission-service';
import { getLogger } from '../../../../utils/logger';

const defaultLog = getLogger('paths/dwc/eml/{datasetId}/handlebar');
Expand Down Expand Up @@ -60,18 +59,18 @@ export function getHandleBarsTemplateByDatasetId(): RequestHandler {
return async (req, res) => {
const connection = req['keycloak_token'] ? getDBConnection(req['keycloak_token']) : getAPIUserDBConnection();

const datasetId = String(req.params.datasetId);
// const datasetId = String(req.params.datasetId);

try {
await connection.open();

const submissionService = new SubmissionService(connection);
// const submissionService = new SubmissionService(connection);

const result = await submissionService.getHandleBarsTemplateByDatasetId(datasetId);
// const result = await submissionService.getHandleBarsTemplateByDatasetId(datasetId);

await connection.commit();

res.status(200).json(result);
res.status(200).json('');
} catch (error) {
defaultLog.error({ label: 'getHandleBarsTemplateByDatasetId', message: 'error', error });
await connection.rollback();
Expand Down
6 changes: 6 additions & 0 deletions api/src/paths/submission/intake.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import sinonChai from 'sinon-chai';
import * as db from '../../database/db';
import { HTTPError } from '../../errors/http-error';
import { SystemUser } from '../../repositories/user-repository';
import { RegionService } from '../../services/region-service';
import { SearchIndexService } from '../../services/search-index-service';
import { SubmissionService } from '../../services/submission-service';
import { ValidationService } from '../../services/validation-service';
Expand Down Expand Up @@ -101,6 +102,10 @@ describe('intake', () => {
.stub(SearchIndexService.prototype, 'indexFeaturesBySubmissionId')
.resolves();

const calculateAndAddRegionsForSubmissionStub = sinon
.stub(RegionService.prototype, 'calculateAndAddRegionsForSubmission')
.resolves();

const requestHandler = intake.submissionIntake();

const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
Expand All @@ -118,6 +123,7 @@ describe('intake', () => {
expect(insertSubmissionRecordWithPotentialConflictStub).to.have.been.calledOnce;
expect(insertSubmissionFeatureRecordsStub).to.have.been.calledOnce;
expect(indexFeaturesBySubmissionIdStub).to.have.been.calledOnce;
expect(calculateAndAddRegionsForSubmissionStub).to.have.been.calledOnce;
expect(mockRes.statusValue).to.eql(200);
expect(mockRes.jsonValue).to.eql({ submission_id: 1 });
});
Expand Down
5 changes: 5 additions & 0 deletions api/src/paths/submission/intake.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { HTTP400 } from '../../errors/http-error';
import { defaultErrorResponses } from '../../openapi/schemas/http-responses';
import { ISubmissionFeature } from '../../repositories/submission-repository';
import { authorizeRequestHandler } from '../../request-handlers/security/authorization';
import { RegionService } from '../../services/region-service';
import { SearchIndexService } from '../../services/search-index-service';
import { SubmissionService } from '../../services/submission-service';
import { ValidationService } from '../../services/validation-service';
Expand Down Expand Up @@ -118,6 +119,7 @@ export function submissionIntake(): RequestHandler {
const submissionService = new SubmissionService(connection);
const validationService = new ValidationService(connection);
const searchIndexService = new SearchIndexService(connection);
const regionService = new RegionService(connection);

// validate the submission
if (!(await validationService.validateSubmissionFeatures(submissionFeatures))) {
Expand All @@ -138,6 +140,9 @@ export function submissionIntake(): RequestHandler {
// Index the submission feature record properties
await searchIndexService.indexFeaturesBySubmissionId(response.submission_id);

// Calculate and add submission regions
await regionService.calculateAndAddRegionsForSubmission(response.submission_id, 0.3);

await connection.commit();

res.status(200).json(response);
Expand Down
56 changes: 56 additions & 0 deletions api/src/repositories/region-repository.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import chai, { expect } from 'chai';
import { describe } from 'mocha';
import { QueryResult } from 'pg';
import sinon from 'sinon';
import sinonChai from 'sinon-chai';
import { getMockDBConnection } from '../__mocks__/db';
import { RegionRepository } from './region-repository';

chai.use(sinonChai);

describe('RegionRepository', () => {
describe('getRegions', () => {
afterEach(() => {
sinon.restore();
});

it('returns an array of region records', async () => {
const mockQueryResponse = { rowCount: 1, rows: [{ region_id: 1 }] } as any as Promise<QueryResult<any>>;
const connection = getMockDBConnection({
sql: () => mockQueryResponse
});

const repo = new RegionRepository(connection);

const regions = await repo.getRegions();
expect(regions.length).to.greaterThanOrEqual(1);
});
});

describe('calculateRegionsForASubmission', () => {
it('should succeed without issue', async () => {
const mockQueryResponse = { rowCount: 1, rows: [{ region_id: 1 }] } as any as Promise<QueryResult<any>>;
const connection = getMockDBConnection({
sql: () => mockQueryResponse
});
const repo = new RegionRepository(connection);

const regions = await repo.calculateRegionsForASubmission(1);

expect(regions).to.be.eql([{ region_id: 1 }]);
});
});

describe('insertSubmissionRegions', () => {
it('should return early with no regions', async () => {
const connection = getMockDBConnection({
sql: sinon.mock()
});

const repo = new RegionRepository(connection);
await repo.insertSubmissionRegions(1, []);

expect(connection.sql).to.not.be.called;
});
});
});
99 changes: 99 additions & 0 deletions api/src/repositories/region-repository.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import SQL from 'sql-template-strings';
import { z } from 'zod';
import { getKnex } from '../database/db';
import { BaseRepository } from './base-repository';

export const RegionRecord = z.object({
region_id: z.any(),
region_name: z.string(),
org_unit: z.string(),
org_unit_name: z.string(),
object_id: z.number(),
feature_code: z.string(),
feature_name: z.string(),
geojson: z.any(),
geometry: z.any(),
geography: z.any(),
create_date: z.string(),
create_user: z.number(),
update_date: z.string().nullable(),
update_user: z.number().nullable(),
revision_count: z.number()
});
export type RegionRecord = z.infer<typeof RegionRecord>;

/**
* A repository class for accessing region data.
*
* @export
* @class RegionRepository
* @extends {BaseRepository}
*/
export class RegionRepository extends BaseRepository {
/**
* Fetches all region records.
*
* @returns {*} {Promise<RegionRecord[]>} An array of Region Records
* @memberof RegionRepository
*/
async getRegions(): Promise<RegionRecord[]> {
const sql = SQL`
SELECT * FROM region_lookup
`;
const response = await this.connection.sql(sql, RegionRecord);
return response.rows;
}

/**
* Calculates region intersects for a submission search_spatial data.
* Submission spatial data is collected then converted into a single polygon using ST_ConvexHull (https://postgis.net/docs/ST_ConvexHull.html)
* Intersections are calculated based on area coverage passed in through intersectionThreshold. Area calculation done using ST_Area (https://postgis.net/docs/ST_Area.html)
* Any regions intersecting with this calculated value are returned.
*
* intersectThreshold is expecting a range of values from 0.0 - 1.0.
* A value of 0.0 means 0% of the geometries area need to intersect meaning all values from `region_lookup` will be returned.
* A value of 1.0 means 100% of the geometries area need to be an exact match before returning a value.
* A value of 0.3 means that 30% of the geometries area need to intersect before returning a value.
*
* @param {number} submissionId
* @param {number} [intersectThreshold=1] intersectThreshold Expected 0.0 - 1.0. Determines the percentage threshold for intersections to be valid
* @returns {*} {Promise<{region_id: number}}[]>} An array of found region ids
* @memberof RegionRepository
*/
async calculateRegionsForASubmission(submissionId: number, intersectThreshold = 1): Promise<{ region_id: number }[]> {
const sql = SQL`
SELECT rl.region_id , rl.region_name
FROM region_lookup rl
WHERE fn_calculate_area_intersect(rl.geography::geometry, (
SELECT ST_ConvexHull(st_collect(ss.value::geometry))
FROM search_spatial ss, submission_feature sf
WHERE ss.submission_feature_id = sf.submission_feature_id
AND sf.submission_id = ${submissionId}
)::geometry, ${intersectThreshold})
GROUP BY rl.region_name, rl.region_id;
`;
const response = await this.connection.sql(sql);
return response.rows;
}

/**
* Associates submissions with regions. This function quits early if no regions are provided.
*
* @param {number} submissionId
* @param {number[]} regionIds
* @memberof RegionRepository
*/
async insertSubmissionRegions(submissionId: number, regionIds: { region_id: number }[]) {
// no regions, exit early
if (!regionIds.length) {
return;
}

const sql = getKnex()
.queryBuilder()
.into('submission_regions')
.insert(regionIds.map(({ region_id }) => ({ region_id, submission_id: submissionId })));

await this.connection.knex(sql);
}
}
Loading

0 comments on commit 9084dec

Please sign in to comment.