From 419f67202ddc40812bda6fd4784ecc46edc2d046 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 5 Dec 2023 14:29:14 -0800 Subject: [PATCH 01/37] Squashed SIMSBIOHUB-365 --- api/package-lock.json | 5 + api/package.json | 2 + api/src/database/db.test.ts | 2 +- api/src/database/db.ts | 2 +- api/src/openapi/root-api-doc.ts | 28 ++++ .../openapi/schemas/biohub-data-submission.ts | 51 +++---- api/src/paths/dataset/intake.ts | 137 ++++++++++++++++++ .../submission-repository.test.ts | 10 +- api/src/repositories/submission-repository.ts | 102 +++++++++++-- api/src/repositories/validation-repository.ts | 47 ++++++ api/src/services/artifact-service.test.ts | 22 ++- api/src/services/artifact-service.ts | 12 +- api/src/services/submission-service.test.ts | 5 +- api/src/services/submission-service.ts | 35 ++++- api/src/services/validation-service.ts | 109 +++++++++++++- database/package-lock.json | 114 +++++++++++---- .../src/seeds/02_populate_feature_tables.ts | 18 ++- 17 files changed, 582 insertions(+), 119 deletions(-) create mode 100644 api/src/paths/dataset/intake.ts diff --git a/api/package-lock.json b/api/package-lock.json index 7e0745025..80f92bca8 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -1592,6 +1592,11 @@ "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" }, + "avj": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/avj/-/avj-0.0.0.tgz", + "integrity": "sha512-uYMMuRd+Ux8xH8L1NnAMy+aTsV+UBgQbS3ckRi3ERZWwq5eyXC5D3lC+8w/ZEjHhNpgUUCu61zJktMS8wse+Mg==" + }, "aws-sdk": { "version": "2.1398.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1398.0.tgz", diff --git a/api/package.json b/api/package.json index 216e92f37..904b21233 100644 --- a/api/package.json +++ b/api/package.json @@ -31,6 +31,7 @@ "@elastic/elasticsearch": "~8.1.0", "adm-zip": "~0.5.5", "ajv": "~8.6.3", + "avj": "0.0.0", "aws-sdk": "^2.1391.0", "axios": "~0.21.4", "clamdjs": "~1.0.2", @@ -41,6 +42,7 @@ "fast-json-patch": "~3.1.1", "fast-xml-parser": "~4.1.3", "fastq": "^1.15.0", + "json-schema-traverse": "^1.0.0", "jsonpath-plus": "~7.2.0", "jsonwebtoken": "~8.5.1", "jwks-rsa": "~2.0.5", diff --git a/api/src/database/db.test.ts b/api/src/database/db.test.ts index 406d6c1c5..37e168882 100644 --- a/api/src/database/db.test.ts +++ b/api/src/database/db.test.ts @@ -421,7 +421,7 @@ describe('db', () => { db.getServiceAccountDBConnection(SOURCE_SYSTEM['SIMS-SVC-4464']); expect(getDBConnectionStub).to.have.been.calledWith({ - preferred_username: `service-account-${SOURCE_SYSTEM['SIMS-SVC-4464']}@${SYSTEM_IDENTITY_SOURCE.SYSTEM}`, + preferred_username: `service-account-${SOURCE_SYSTEM['SIMS-SVC-4464']}`, identity_provider: SYSTEM_IDENTITY_SOURCE.SYSTEM }); diff --git a/api/src/database/db.ts b/api/src/database/db.ts index 1b9fcccca..680227de4 100644 --- a/api/src/database/db.ts +++ b/api/src/database/db.ts @@ -421,7 +421,7 @@ export const getAPIUserDBConnection = (): IDBConnection => { */ export const getServiceAccountDBConnection = (sourceSystem: SOURCE_SYSTEM): IDBConnection => { return getDBConnection({ - preferred_username: `service-account-${sourceSystem}@${SYSTEM_IDENTITY_SOURCE.SYSTEM}`, + preferred_username: `service-account-${sourceSystem}`, identity_provider: SYSTEM_IDENTITY_SOURCE.SYSTEM }); }; diff --git a/api/src/openapi/root-api-doc.ts b/api/src/openapi/root-api-doc.ts index 34576a584..07a7ea875 100644 --- a/api/src/openapi/root-api-doc.ts +++ b/api/src/openapi/root-api-doc.ts @@ -140,6 +140,34 @@ export const rootAPIDoc = { } } } + }, + SubmissionFeature: { + title: 'BioHub Data Submission Feature', + type: 'object', + required: ['id', 'type', 'properties', 'features'], + properties: { + id: { + title: 'Unique id of the feature', + type: 'string' + }, + type: { + title: 'Feature type', + type: 'string' + }, + properties: { + title: 'Feature properties', + type: 'object', + properties: {} + }, + features: { + title: 'Feature child features', + type: 'array', + items: { + $ref: '#/components/schemas/SubmissionFeature' + } + } + }, + additionalProperties: false } } } diff --git a/api/src/openapi/schemas/biohub-data-submission.ts b/api/src/openapi/schemas/biohub-data-submission.ts index 1d06a5cba..92508a381 100644 --- a/api/src/openapi/schemas/biohub-data-submission.ts +++ b/api/src/openapi/schemas/biohub-data-submission.ts @@ -1,7 +1,7 @@ export const BioHubDataSubmission = { title: 'BioHub Data Submission', type: 'object', - required: ['id', 'type', 'features'], + required: ['id', 'type', 'properties', 'features'], properties: { id: { title: 'Unique id of the submission', @@ -19,40 +19,27 @@ export const BioHubDataSubmission = { features: { type: 'array', items: { - $ref: '#/$defs/Feature' - } - } - }, - $defs: { - Feature: { - title: 'BioHub Data Submission Feature', - type: 'object', - required: ['id', 'type', 'properties', 'features'], - properties: { - id: { - title: 'Unique id of the feature', - type: 'string' - }, - type: { - title: 'Feature type', - type: 'string' - }, + title: 'BioHub Data Submission Feature', + type: 'object', + required: ['id', 'type', 'properties', 'features'], properties: { - title: 'Feature properties', - type: 'object', - properties: {} - }, - features: { - title: 'Feature child features', - type: 'array', - items: { - $ref: '#/$defs/Feature' + id: { + title: 'Unique id of the feature', + type: 'string' + }, + type: { + title: 'Feature type', + type: 'string', + enum: ['observation'] + }, + properties: { + title: 'Feature properties', + type: 'object', + properties: {} } } - }, - additionalProperties: false - }, - additionalProperties: false + } + } }, additionalProperties: false }; diff --git a/api/src/paths/dataset/intake.ts b/api/src/paths/dataset/intake.ts new file mode 100644 index 000000000..dbc6907ad --- /dev/null +++ b/api/src/paths/dataset/intake.ts @@ -0,0 +1,137 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { SOURCE_SYSTEM } from '../../constants/database'; +import { getServiceAccountDBConnection } from '../../database/db'; +import { HTTP400 } from '../../errors/http-error'; +import { defaultErrorResponses } from '../../openapi/schemas/http-responses'; +import { authorizeRequestHandler } from '../../request-handlers/security/authorization'; +import { SubmissionService } from '../../services/submission-service'; +import { ValidationService } from '../../services/validation-service'; +import { getKeycloakSource } from '../../utils/keycloak-utils'; +import { getLogger } from '../../utils/logger'; + +const defaultLog = getLogger('paths/dataset/intake'); + +export const POST: Operation = [ + authorizeRequestHandler(() => { + return { + and: [ + { + validServiceClientIDs: [SOURCE_SYSTEM['SIMS-SVC-4464']], + discriminator: 'ServiceClient' + } + ] + }; + }), + datasetIntake() +]; + +POST.apiDoc = { + description: 'Submit dataset to BioHub', + tags: ['dataset'], + security: [ + { + Bearer: [] + } + ], + requestBody: { + content: { + 'application/json': { + schema: { + title: 'BioHub Data Submission', + type: 'object', + required: ['id', 'type', 'properties', 'features'], + properties: { + id: { + title: 'Unique id of the submission', + type: 'string' + }, + type: { + type: 'string', + enum: ['dataset'] + }, + properties: { + title: 'Dataset properties', + type: 'object', + properties: {} + }, + features: { + type: 'array', + items: { + $ref: '#/components/schemas/SubmissionFeature' + }, + additionalProperties: false + } + }, + additionalProperties: false + } + } + } + }, + responses: { + 200: { + description: 'Submission OK', + content: { + 'application/json': { + schema: { + type: 'object', + required: ['submission_id'], + properties: { + submission_id: { + type: 'integer' + } + } + } + } + } + }, + ...defaultErrorResponses + } +}; + +export function datasetIntake(): RequestHandler { + return async (req, res) => { + const sourceSystem = getKeycloakSource(req['keycloak_token']); + + if (!sourceSystem) { + throw new HTTP400('Failed to identify known submission source system', [ + 'token did not contain a clientId/azp or clientId/azp value is unknown' + ]); + } + + const dataset = { + ...req.body, + properties: { ...req.body.properties, additionalInformation: req.body.properties.additionalInformation } + }; + const id = req.body.id; + + const connection = getServiceAccountDBConnection(sourceSystem); + + try { + await connection.open(); + + const submissionService = new SubmissionService(connection); + const validationService = new ValidationService(connection); + + // validate the dataset submission + if (!(await validationService.validateDatasetSubmission(dataset))) { + throw new HTTP400('Invalid dataset submission'); + } + + // insert the submission record + const response = await submissionService.insertSubmissionRecordWithPotentialConflict(id); + + // insert each submission feature record + await submissionService.insertSubmissionFeatureRecords(response.submission_id, dataset.features); + + await connection.commit(); + res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'datasetIntake', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/repositories/submission-repository.test.ts b/api/src/repositories/submission-repository.test.ts index 5512d4ee9..f994a4f5b 100644 --- a/api/src/repositories/submission-repository.test.ts +++ b/api/src/repositories/submission-repository.test.ts @@ -490,10 +490,7 @@ describe('SubmissionRepository', () => { const submissionRepository = new SubmissionRepository(mockDBConnection); - const response = await submissionRepository.insertSubmissionRecordWithPotentialConflict({ - uuid: 'aaaa', - source_transform_id: 1 - }); + const response = await submissionRepository.insertSubmissionRecordWithPotentialConflict('aaaa'); expect(response).to.eql({ uuid: 'aaaa', @@ -510,10 +507,7 @@ describe('SubmissionRepository', () => { const submissionRepository = new SubmissionRepository(mockDBConnection); try { - await submissionRepository.insertSubmissionRecordWithPotentialConflict({ - uuid: 'bbbb', - source_transform_id: 3 - }); + await submissionRepository.insertSubmissionRecordWithPotentialConflict('aaaa'); expect.fail(); } catch (actualError) { expect((actualError as ApiExecuteSQLError).message).to.equal('Failed to get or insert submission record'); diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts index 0055fd0a5..e7d076730 100644 --- a/api/src/repositories/submission-repository.ts +++ b/api/src/repositories/submission-repository.ts @@ -19,6 +19,19 @@ export interface IDatasetsForReview { keywords: string[]; } +export interface IFeatureSubmission { + id: string; + type: string; + properties: object; +} + +export interface IDatasetSubmission { + id: string; + type: string; + properties: object; + features: IFeatureSubmission[]; +} + export const DatasetMetadata = z.object({ dataset_id: z.string(), submission_id: z.number(), @@ -238,22 +251,20 @@ export class SubmissionRepository extends BaseRepository { * uuid with the given value in the case that they match, which allows us to retrieve the submission_id * and infer that the query ran successfully. * - * @param {ISubmissionModel} submissionData The submission record - * @return {*} {Promise<{ submission_id: number }>} The primary key of the submission + * @param {string} uuid + * @return {*} {Promise<{ submission_id: number }>} * @memberof SubmissionRepository */ - async insertSubmissionRecordWithPotentialConflict( - submissionData: ISubmissionModel - ): Promise<{ submission_id: number }> { + async insertSubmissionRecordWithPotentialConflict(uuid: string): Promise<{ submission_id: number }> { const sqlStatement = SQL` INSERT INTO submission ( - source_transform_id, - uuid + uuid, + publish_timestamp ) VALUES ( - ${submissionData.source_transform_id}, - ${submissionData.uuid} + ${uuid}, + now() ) - ON CONFLICT (uuid) DO UPDATE SET uuid = ${submissionData.uuid} + ON CONFLICT (uuid) DO UPDATE SET publish_timestamp = now() RETURNING submission_id; `; @@ -270,6 +281,77 @@ export class SubmissionRepository extends BaseRepository { return response.rows[0]; } + /** + * Insert a new submission feature record. + * + * @param {number} submissionId + * @param {IFeatureSubmission} feature + * @param {number} featureTypeId + * @return {*} {Promise<{ submission_feature_id: number }>} + * @memberof SubmissionRepository + */ + async insertSubmissionFeatureRecord( + submissionId: number, + featureTypeId: number, + feature: IFeatureSubmission + ): Promise<{ submission_feature_id: number }> { + const sqlStatement = SQL` + INSERT INTO submission_feature ( + submission_id, + feature_type_id, + data, + record_effective_date + ) VALUES ( + ${submissionId}, + ${featureTypeId}, + ${feature}, + now() + ) + RETURNING + submission_feature_id; + `; + + const response = await this.connection.sql<{ submission_feature_id: number }>(sqlStatement); + + if (response.rowCount !== 1) { + throw new ApiExecuteSQLError('Failed to insert submission feature record', [ + 'SubmissionRepository->insertSubmissionFeatureRecord', + 'rowCount was null or undefined, expected rowCount = 1' + ]); + } + + return response.rows[0]; + } + + /** + * Get feature type id by name. + * + * @param {string} name + * @return {*} {Promise<{ feature_type_id: number }>} + * @memberof SubmissionRepository + */ + async getFeatureTypeIdByName(name: string): Promise<{ feature_type_id: number }> { + const sqlStatement = SQL` + SELECT + feature_type_id + FROM + feature_type + WHERE + name = ${name}; + `; + + const response = await this.connection.sql<{ feature_type_id: number }>(sqlStatement); + + if (!response.rowCount) { + throw new ApiExecuteSQLError('Failed to get feature type record', [ + 'SubmissionRepository->getFeatureTypeByName', + 'rowCount was null or undefined, expected rowCount != 0' + ]); + } + + return response.rows[0]; + } + /** * Update the `eml_source` column of a submission record. * diff --git a/api/src/repositories/validation-repository.ts b/api/src/repositories/validation-repository.ts index f45a739de..b5a3b632c 100644 --- a/api/src/repositories/validation-repository.ts +++ b/api/src/repositories/validation-repository.ts @@ -10,6 +10,13 @@ export interface IStyleModel { something: any; //TODO } +export interface IFeatureProperties { + name: string; + display_name: string; + description: string; + type: string; +} + /** *THIS REPO IS ALL HARD CODED DO NOT USE * @@ -18,6 +25,46 @@ export interface IStyleModel { * @extends {BaseRepository} */ export class ValidationRepository extends BaseRepository { + /** + * Get Feature properties for given feature type + * + * @param {string} featureType + * @return {*} {Promise} + * @memberof ValidationRepository + */ + async getFeatureValidationProperties(featureType: string): Promise { + const sqlStatement = SQL` + SELECT + fp.name, + fp.display_name, + fp.description, + fpt.name as type + FROM + feature_type_property ftp + LEFT JOIN + feature_property fp + ON + ftp.feature_property_id = fp.feature_property_id + LEFT JOIN + feature_property_type fpt + ON + fp.feature_property_type_id = fpt.feature_property_type_id + WHERE + feature_type_id = (select feature_type_id from feature_type ft where ft.name = ${featureType}); + `; + + const response = await this.connection.sql(sqlStatement); + + if (response.rowCount === 0) { + throw new ApiExecuteSQLError('Failed to get dataset validation properties', [ + 'ValidationRepository->getFeatureValidationProperties', + 'rowCount was null or undefined, expected rowCount != 0' + ]); + } + + return response.rows; + } + /** * Insert Style sheet into db * diff --git a/api/src/services/artifact-service.test.ts b/api/src/services/artifact-service.test.ts index 5f04938ef..9cb2410b6 100644 --- a/api/src/services/artifact-service.test.ts +++ b/api/src/services/artifact-service.test.ts @@ -6,7 +6,6 @@ import sinonChai from 'sinon-chai'; import { HTTPError } from '../errors/http-error'; import { Artifact, ArtifactMetadata, ArtifactRepository } from '../repositories/artifact-repository'; import { SecurityRepository } from '../repositories/security-repository'; -import { ISourceTransformModel } from '../repositories/submission-repository'; import * as file_utils from '../utils/file-utils'; import { getMockDBConnection } from '../__mocks__/db'; import { ArtifactService } from './artifact-service'; @@ -95,9 +94,9 @@ describe('ArtifactService', () => { const mockDBConnection = getMockDBConnection({ systemUserId: () => 20 }); const artifactService = new ArtifactService(mockDBConnection); - const transformRecordStub = sinon - .stub(SubmissionService.prototype, 'getSourceTransformRecordBySystemUserId') - .resolves({ source_transform_id: 60 } as unknown as ISourceTransformModel); + // const transformRecordStub = sinon + // .stub(SubmissionService.prototype, 'getSourceTransformRecordBySystemUserId') + // .resolves({ source_transform_id: 60 } as unknown as ISourceTransformModel); // const getOrInsertSubmissionRecordStub = sinon @@ -115,7 +114,7 @@ describe('ArtifactService', () => { await artifactService.uploadAndPersistArtifact(mockDataPackageId, mockArtifactMetadata, mockFileUuid, mockFile); expect.fail(); } catch (actualError) { - expect(transformRecordStub).to.be.calledWith(20); + // expect(transformRecordStub).to.be.calledWith(20); expect((actualError as HTTPError).message).to.equal('Test upload failed'); expect(insertRecordStub).to.not.be.called; } @@ -125,9 +124,9 @@ describe('ArtifactService', () => { const mockDBConnection = getMockDBConnection({ systemUserId: () => 20 }); const artifactService = new ArtifactService(mockDBConnection); - const transformRecordStub = sinon - .stub(SubmissionService.prototype, 'getSourceTransformRecordBySystemUserId') - .resolves({ source_transform_id: 60 } as unknown as ISourceTransformModel); + // const transformRecordStub = sinon + // .stub(SubmissionService.prototype, 'getSourceTransformRecordBySystemUserId') + // .resolves({ source_transform_id: 60 } as unknown as ISourceTransformModel); const insertSubmissionRecordWithPotentialConflictStub = sinon .stub(SubmissionService.prototype, 'insertSubmissionRecordWithPotentialConflict') @@ -145,12 +144,9 @@ describe('ArtifactService', () => { await artifactService.uploadAndPersistArtifact(mockDataPackageId, mockArtifactMetadata, mockFileUuid, mockFile); expect.fail(); } catch (actualError) { - expect(transformRecordStub).to.be.calledWith(20); + // expect(transformRecordStub).to.be.calledWith(20); - expect(insertSubmissionRecordWithPotentialConflictStub).to.be.calledWith({ - source_transform_id: 60, - uuid: mockDataPackageId - }); + expect(insertSubmissionRecordWithPotentialConflictStub).to.be.calledWith(mockDataPackageId); expect(getNextArtifactIdsStub).to.be.calledWith(); expect(uploadStub).to.be.calledWith( mockFile, diff --git a/api/src/services/artifact-service.ts b/api/src/services/artifact-service.ts index 03c6bef6b..782a1ea06 100644 --- a/api/src/services/artifact-service.ts +++ b/api/src/services/artifact-service.ts @@ -67,10 +67,11 @@ export class ArtifactService extends DBService { ): Promise<{ artifact_id: number }> { defaultLog.debug({ label: 'uploadAndPersistArtifact' }); + // NOTE: Disabled for now, as we are not using the source transform record // Fetch the source transform record for this submission based on the source system user id - const sourceTransformRecord = await this.submissionService.getSourceTransformRecordBySystemUserId( - this.connection.systemUserId() - ); + // const sourceTransformRecord = await this.submissionService.getSourceTransformRecordBySystemUserId( + // this.connection.systemUserId() + // ); // Retrieve the next artifact primary key assigned to this artifact once it is inserted const artifact_id = (await this.getNextArtifactIds())[0]; @@ -83,10 +84,7 @@ export class ArtifactService extends DBService { }); // Create a new submission for the artifact collection - const { submission_id } = await this.submissionService.insertSubmissionRecordWithPotentialConflict({ - source_transform_id: sourceTransformRecord.source_transform_id, - uuid: dataPackageId - }); + const { submission_id } = await this.submissionService.insertSubmissionRecordWithPotentialConflict(dataPackageId); // Upload the artifact to S3 await uploadFileToS3(file, s3Key, { filename: file.originalname }); diff --git a/api/src/services/submission-service.test.ts b/api/src/services/submission-service.test.ts index fc32c792f..26b63d33c 100644 --- a/api/src/services/submission-service.test.ts +++ b/api/src/services/submission-service.test.ts @@ -50,10 +50,7 @@ describe('SubmissionService', () => { .stub(SubmissionRepository.prototype, 'insertSubmissionRecordWithPotentialConflict') .resolves({ submission_id: 1 }); - const response = await submissionService.insertSubmissionRecordWithPotentialConflict({ - uuid: '', - source_transform_id: 1 - }); + const response = await submissionService.insertSubmissionRecordWithPotentialConflict('aaaa'); expect(repo).to.be.calledOnce; expect(response).to.be.eql({ submission_id: 1 }); diff --git a/api/src/services/submission-service.ts b/api/src/services/submission-service.ts index e97472a08..55cd4434f 100644 --- a/api/src/services/submission-service.ts +++ b/api/src/services/submission-service.ts @@ -5,6 +5,7 @@ import { IDBConnection } from '../database/db'; import { ApiExecuteSQLError } from '../errors/api-error'; import { IDatasetsForReview, + IFeatureSubmission, IHandlebarsTemplates, ISourceTransformModel, ISubmissionJobQueueRecord, @@ -51,15 +52,39 @@ export class SubmissionService extends DBService { /** * Insert a new submission record, returning the record having the matching UUID if it already exists + * in the database. * - * @param {ISubmissionModel} submissionData + * @param {string} uuid * @return {*} {Promise<{ submission_id: number }>} * @memberof SubmissionService */ - async insertSubmissionRecordWithPotentialConflict( - submissionData: ISubmissionModel - ): Promise<{ submission_id: number }> { - return this.submissionRepository.insertSubmissionRecordWithPotentialConflict(submissionData); + async insertSubmissionRecordWithPotentialConflict(uuid: string): Promise<{ submission_id: number }> { + return this.submissionRepository.insertSubmissionRecordWithPotentialConflict(uuid); + } + + /** + * insert submission feature record + * + * @param {number} submissionId + * @param {IFeatureSubmission[]} submissionFeature + * @return {*} {Promise<{ submission_feature_id: number }[]>} + * @memberof SubmissionService + */ + async insertSubmissionFeatureRecords( + submissionId: number, + submissionFeature: IFeatureSubmission[] + ): Promise<{ submission_feature_id: number }[]> { + const promise = submissionFeature.map(async (feature) => { + const featureTypeId = await this.submissionRepository.getFeatureTypeIdByName(feature.type); + + return this.submissionRepository.insertSubmissionFeatureRecord( + submissionId, + featureTypeId.feature_type_id, + feature + ); + }); + + return Promise.all(promise); } /** diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index f76387682..dc476a120 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -1,5 +1,11 @@ import { IDBConnection } from '../database/db'; -import { IInsertStyleSchema, IStyleModel, ValidationRepository } from '../repositories/validation-repository'; +import { IDatasetSubmission } from '../repositories/submission-repository'; +import { + IFeatureProperties, + IInsertStyleSchema, + IStyleModel, + ValidationRepository +} from '../repositories/validation-repository'; import { ICsvState } from '../utils/media/csv/csv-file'; import { DWCArchive } from '../utils/media/dwc/dwc-archive-file'; import { IMediaState } from '../utils/media/media-file'; @@ -8,11 +14,112 @@ import { DBService } from './db-service'; export class ValidationService extends DBService { validationRepository: ValidationRepository; + validationProperties: Map; constructor(connection: IDBConnection) { super(connection); this.validationRepository = new ValidationRepository(connection); + this.validationProperties = new Map(); + } + + /** + * Validate dataset submission + * + * @param {IDatasetSubmission} dataset + * @return {*} {Promise} + * @memberof ValidationService + */ + async validateDatasetSubmission(dataset: IDatasetSubmission): Promise { + // validate dataset.type is 'dataset' + const datasetFeatureType = dataset.type; + + // get dataset validation properties + const datasetValidationProperties = await this.getFeatureValidationProperties(datasetFeatureType); + + // get features in dataset + const features = dataset.features; + + try { + // validate dataset properties + await this.validateProperties(datasetValidationProperties, dataset.properties); + + // validate features + for (const feature of features) { + const featureType = feature.type; + + // get feature validation properties + const featureValidationProperties = await this.getFeatureValidationProperties(featureType); + + // validate feature properties + await this.validateProperties(featureValidationProperties, feature.properties); + } + } catch (e) { + console.log(e); + return false; + } + return true; + } + + async validateProperties(properties: IFeatureProperties[], dataProperties: any): Promise { + const throwPropertyError = (property: IFeatureProperties) => { + throw new Error(`Property ${property.name} is not of type ${property.type}`); + }; + + for (const property of properties) { + const dataProperty = dataProperties[property.name]; + + if (!dataProperty) { + throw new Error(`Property ${property.name} not found in data`); + } + + if (property.type === 'string') { + if (typeof dataProperty !== 'string') { + throwPropertyError(property); + } + } else if (property.type === 'number') { + if (typeof dataProperty !== 'number') { + throwPropertyError(property); + } + } else if (property.type === 'boolean') { + if (typeof dataProperty !== 'boolean') { + throwPropertyError(property); + } + } else if (property.type === 'object') { + if (typeof dataProperty !== 'object') { + throwPropertyError(property); + } + } else if (property.type === 'spatial') { + if (Array.isArray(dataProperty) === false) { + throwPropertyError(property); + } + } else if (property.type === 'datetime') { + if (typeof dataProperty !== 'string') { + throwPropertyError(property); + } + + const date = new Date(dataProperty); + + if (date.toString() === 'Invalid Date') { + throw new Error(`Property ${property.name} is not a valid date`); + } + } else { + throw new Error(`Property ${property.name} has an invalid type`); + } + } + + return true; + } + + async getFeatureValidationProperties(featureType: string): Promise { + if (this.validationProperties.get(featureType) === undefined) { + this.validationProperties.set( + featureType, + await this.validationRepository.getFeatureValidationProperties(featureType) + ); + } + + return this.validationProperties.get(featureType) as IFeatureProperties[]; } /** diff --git a/database/package-lock.json b/database/package-lock.json index 2148757cf..b6fd6896a 100644 --- a/database/package-lock.json +++ b/database/package-lock.json @@ -503,7 +503,7 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, "colorette": { @@ -519,7 +519,25 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "dev": true }, "core-util-is": { @@ -927,7 +945,7 @@ "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, "fastq": { @@ -995,7 +1013,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, "function-bind": { @@ -1018,7 +1036,7 @@ "functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", "dev": true }, "functions-have-names": { @@ -1172,7 +1190,7 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true }, "has-property-descriptors": { @@ -1238,13 +1256,13 @@ "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "dev": true, "requires": { "once": "^1.3.0", @@ -1287,7 +1305,7 @@ "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true }, "is-bigint": { @@ -1335,7 +1353,7 @@ "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true }, "is-fullwidth-code-point": { @@ -1442,7 +1460,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, "js-tokens": { @@ -1476,7 +1494,25 @@ "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true + }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "dev": true + }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "dev": true + }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", "dev": true }, "jsonparse": { @@ -1519,7 +1555,7 @@ "load-json-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", - "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "integrity": "sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==", "dev": true, "requires": { "graceful-fs": "^4.1.2", @@ -1542,7 +1578,7 @@ "lodash.truncate": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", - "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=", + "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true }, "lru-cache": { @@ -1563,7 +1599,7 @@ "memorystream": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", - "integrity": "sha1-htcJCzDORV1j+64S3aUaR93K+bI=", + "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==", "dev": true }, "merge2": { @@ -1599,7 +1635,7 @@ "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, "nice-try": { @@ -1670,7 +1706,7 @@ "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", "dev": true }, "semver": { @@ -1682,7 +1718,7 @@ "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", "dev": true, "requires": { "shebang-regex": "^1.0.0" @@ -1691,7 +1727,7 @@ "shebang-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", "dev": true }, "which": { @@ -1732,7 +1768,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, "requires": { "wrappy": "1" @@ -1769,7 +1805,7 @@ "parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", "dev": true, "requires": { "error-ex": "^1.3.1", @@ -1779,7 +1815,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "dev": true }, "path-key": { @@ -1869,7 +1905,7 @@ "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", "dev": true }, "postgres-array": { @@ -1880,7 +1916,7 @@ "postgres-bytea": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" }, "postgres-date": { "version": "1.0.7", @@ -1949,7 +1985,7 @@ "read-pkg": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", - "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "integrity": "sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==", "dev": true, "requires": { "load-json-file": "^4.0.0", @@ -2073,7 +2109,7 @@ "semver": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", - "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + "integrity": "sha512-VyFUffiBx8hABJ9HYSTXLRwyZtdDHMzMtFmID1aiNAD2BZppBmJm0Hqw3p2jkgxP9BNt1pQ9RnC49P0EcXf6cA==" }, "shebang-command": { "version": "2.0.0", @@ -2190,7 +2226,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, "string-width": { @@ -2269,7 +2305,7 @@ "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", "dev": true }, "strip-json-comments": { @@ -2333,7 +2369,25 @@ "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "dev": true }, "through": { @@ -2518,7 +2572,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, "xtend": { diff --git a/database/src/seeds/02_populate_feature_tables.ts b/database/src/seeds/02_populate_feature_tables.ts index 54443eecd..3c0b11cda 100644 --- a/database/src/seeds/02_populate_feature_tables.ts +++ b/database/src/seeds/02_populate_feature_tables.ts @@ -36,6 +36,8 @@ export async function seed(knex: Knex): Promise { insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('end_date', 'End Date', 'The end date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('geometry', 'Geometry', 'The location of the record', (select feature_property_type_id from feature_property_type where name = 'spatial'), null, now()); insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('count', 'Count', 'The count of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('latitude', 'Latitude', 'The latitude of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('longitude', 'Longitude', 'The longitude of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); -- populate feature_type table insert into feature_type (name, display_name, description, record_effective_date) values ('dataset', 'Dataset', 'A related collection of data (ie: survey)', now()); @@ -49,9 +51,9 @@ export async function seed(knex: Knex): Promise { -- populate feature_type_property table -- feature_type: dataset insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'name'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'description'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'date_range'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'description'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'date_range'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'start_date'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'end_date'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'geometry'), now()); @@ -72,10 +74,12 @@ export async function seed(knex: Knex): Promise { -- feature_type: observation insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'date_range'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'start_date'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'end_date'), now()); - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'geometry'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'date_range'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'start_date'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'end_date'), now()); + -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'geometry'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'latitude'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'longitude'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'count'), now()); -- feature_type: animal From 450f950a0929f3635808f3b3eb5b0f5aa19e12a4 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 5 Dec 2023 14:51:27 -0800 Subject: [PATCH 02/37] SIMSBIOHUB-379: initialize search-index repo --- .../repositories/search-index-respository.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 api/src/repositories/search-index-respository.ts diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts new file mode 100644 index 000000000..55cf7a926 --- /dev/null +++ b/api/src/repositories/search-index-respository.ts @@ -0,0 +1,45 @@ +import { z } from "zod"; +import { getLogger } from "../utils/logger"; +import { BaseRepository } from "./base-repository"; + +const defaultLog = getLogger('repositories/search-index-repository'); + +const SearchableRecord = (valueType: z.ZodType) => z.object({ + search_datetime_id: z.number(), + submission_feature_id: z.number(), + feature_property_id: z.number(), + value: valueType, + create_date: z.date(), + create_user: z.number(), + update_date: z.date().nullable(), + update_user: z.date().nullable(), + revision_count: z.number(), +}); + +// export type SearchableRecordType = z.infer>; +export type SearchableRecordType = T extends z.ZodType ? U : never; + + +type A = SearchableRecordType['create_date'] + + +export const DatetimeSearchableRecord = z.object({ + search_datetime_id: z.number(), + submission_feature_id: z.number(), + feature_property_id: z.number(), + value: z.date(), + create_date: z.date(), + create_user: z.number(), + update_date: z.date().nullable(), + update_user: z.date().nullable(), + revision_count: z.number() +}); + +export type DatetimeSearchableRecord = z.infer; + +export class SearchIndexRepository extends BaseRepository { + + async createSearchableDatetime(datetimeRecord: CreateDatetime) => { + // + } +} From 29dde5505ce9ab137fdc66952f02e0c6378a7291 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 5 Dec 2023 15:38:58 -0800 Subject: [PATCH 03/37] SIMSBIOHUB-379: Finished typedefs for search-index-repo insertion methods --- .../repositories/search-index-respository.ts | 134 +++++++++++++++--- 1 file changed, 116 insertions(+), 18 deletions(-) diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index 55cf7a926..5e1ebc6c8 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -1,14 +1,20 @@ import { z } from "zod"; import { getLogger } from "../utils/logger"; import { BaseRepository } from "./base-repository"; +import { getKnex } from "../database/db"; +import { ApiExecuteSQLError } from "../errors/api-error"; const defaultLog = getLogger('repositories/search-index-repository'); -const SearchableRecord = (valueType: z.ZodType) => z.object({ - search_datetime_id: z.number(), +const Point = z.object({ + type: z.literal('Point'), + coordinates: z.tuple([z.number(), z.number()]), +}); + +const SearchableRecord = z.object({ submission_feature_id: z.number(), feature_property_id: z.number(), - value: valueType, + value: z.unknown(), create_date: z.date(), create_user: z.number(), update_date: z.date().nullable(), @@ -16,30 +22,122 @@ const SearchableRecord = (valueType: z.ZodType) => z.object({ revision_count: z.number(), }); -// export type SearchableRecordType = z.infer>; -export type SearchableRecordType = T extends z.ZodType ? U : never; +type InsertSearchableRecordKey = 'value' | 'feature_property_id' +export const DatetimeSearchableRecord = SearchableRecord.extend({ + search_datetime_id: z.date(), + value: z.date() +}); -type A = SearchableRecordType['create_date'] +export const NumberSearchableRecord = SearchableRecord.extend({ + search_number_id: z.number(), + value: z.number() +}); +export const SpatialSearchableRecord = SearchableRecord.extend({ + search_spatial_id: z.number(), + value: Point +}); -export const DatetimeSearchableRecord = z.object({ - search_datetime_id: z.number(), - submission_feature_id: z.number(), - feature_property_id: z.number(), - value: z.date(), - create_date: z.date(), - create_user: z.number(), - update_date: z.date().nullable(), - update_user: z.date().nullable(), - revision_count: z.number() +export const StringSearchableRecord = SearchableRecord.extend({ + search_string_id: z.number(), + value: z.string() }); +export type SearchableRecord = z.infer; export type DatetimeSearchableRecord = z.infer; +export type NumberSearchableRecord = z.infer; +export type SpatialSearchableRecord = z.infer; +export type StringSearchableRecord = z.infer; + +type InsertDatetimeSearchableRecord = Pick +type InsertNumberSearchableRecord = Pick +type InsertSpatialSearchableRecord = Pick +type InsertStringSearchableRecord = Pick export class SearchIndexRepository extends BaseRepository { - async createSearchableDatetime(datetimeRecord: CreateDatetime) => { - // + async insertSearchableDatetimeRecords(datetimeRecords: InsertDatetimeSearchableRecord[]): Promise { + defaultLog.debug({ label: 'insertSearchableDatetimeRecords' }); + + const queryBuilder = getKnex() + .queryBuilder() + .insert(datetimeRecords) + .into('search_datetime') + .returning('*') + + const response = await this.connection.knex(queryBuilder); + + if (response.rowCount !== datetimeRecords.length) { + throw new ApiExecuteSQLError('Failed to insert searchable datetime records', [ + 'SearchIndexRepository->insertSearchableDatetimeRecords', + 'rowCount did not match number of supplied records to insert' + ]); + } + + return response.rows; + } + + async insertSearchableNumberRecords(numberRecords: InsertNumberSearchableRecord[]): Promise { + defaultLog.debug({ label: 'insertSearchableNumberRecords' }); + + const queryBuilder = getKnex() + .queryBuilder() + .insert(numberRecords) + .into('search_number') + .returning('*') + + const response = await this.connection.knex(queryBuilder); + + if (response.rowCount !== numberRecords.length) { + throw new ApiExecuteSQLError('Failed to insert searchable number records', [ + 'SearchIndexRepository->insertSearchableNumberRecords', + 'rowCount did not match number of supplied records to insert' + ]); + } + + return response.rows; + } + + async insertSearchableSpatialRecords(spatialRecords: InsertSpatialSearchableRecord[]): Promise { + defaultLog.debug({ label: 'insertSearchableSpatialRecords' }); + + const queryBuilder = getKnex() + .queryBuilder() + .insert(spatialRecords) + .into('search_spatial') + .returning('*') + + const response = await this.connection.knex(queryBuilder); + + if (response.rowCount !== spatialRecords.length) { + throw new ApiExecuteSQLError('Failed to insert searchable spatial records', [ + 'SearchIndexRepository->insertSearchableSpatialRecords', + 'rowCount did not match number of supplied records to insert' + ]); + } + + return response.rows; + } + + async insertSearchableStringRecords(stringRecords: InsertStringSearchableRecord[]): Promise { + defaultLog.debug({ label: 'insertSearchableStringRecords' }); + + const queryBuilder = getKnex() + .queryBuilder() + .insert(stringRecords) + .into('search_string') + .returning('*') + + const response = await this.connection.knex(queryBuilder); + + if (response.rowCount !== stringRecords.length) { + throw new ApiExecuteSQLError('Failed to insert searchable string records', [ + 'SearchIndexRepository->insertSearchableStringRecords', + 'rowCount did not match number of supplied records to insert' + ]); + } + + return response.rows; } } From 93115cfc5af0882aab8fca62e19bc5c352e2903d Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Thu, 7 Dec 2023 10:35:00 -0800 Subject: [PATCH 04/37] SIMSBIOHUB-379: Create search indexing endpoint --- api/src/paths/dataset/search-idx.ts | 86 +++++++++++++++++++ api/src/repositories/submission-repository.ts | 12 +++ api/src/services/search-index-service.ts | 25 ++++++ app/src/features/home/HomePage.tsx | 3 + app/src/hooks/api/useSubmissionsApi.ts | 9 +- 5 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 api/src/paths/dataset/search-idx.ts create mode 100644 api/src/services/search-index-service.ts diff --git a/api/src/paths/dataset/search-idx.ts b/api/src/paths/dataset/search-idx.ts new file mode 100644 index 000000000..3ad46d0c5 --- /dev/null +++ b/api/src/paths/dataset/search-idx.ts @@ -0,0 +1,86 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { SOURCE_SYSTEM } from '../../constants/database'; +import { getAPIUserDBConnection, getDBConnection } from '../../database/db'; +import { defaultErrorResponses } from '../../openapi/schemas/http-responses'; +import { authorizeRequestHandler } from '../../request-handlers/security/authorization'; +import { getLogger } from '../../utils/logger'; +import { SearchIndexService } from '../../services/search-index-service'; + +const defaultLog = getLogger('paths/dataset/search-index'); + +export const POST: Operation = [ + authorizeRequestHandler(() => { + return { + and: [ + { + validServiceClientIDs: [SOURCE_SYSTEM['SIMS-SVC-4464']], + discriminator: 'ServiceClient' + } + ] + }; + }), + indexSubmission() +]; + +POST.apiDoc = { + description: 'Index dataset in BioHub', + tags: ['dataset'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + description: 'Submission ID', + in: 'query', + name: 'submissionId', + schema: { + type: 'integer', + minimum: 1 + }, + required: true + } + ], + responses: { + 200: { + description: 'TODO', // TODO + content: { + 'application/json': { + schema: { + // TODO + } + } + } + }, + ...defaultErrorResponses + } +}; + +export function indexSubmission(): RequestHandler { + return async (req, res) => { + + const connection = req['keycloak_token'] ? getDBConnection(req['keycloak_token']) : getAPIUserDBConnection(); + + const submissionId = Number(req.query.submissionId); + + try { + await connection.open(); + + const searchIndexService = new SearchIndexService(connection); + + // Index the submission record + const response = await searchIndexService.indexFeaturesBySubmissionId(submissionId) + + await connection.commit(); + res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'datasetIntake', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts index e7d076730..ed6a9c6cb 100644 --- a/api/src/repositories/submission-repository.ts +++ b/api/src/repositories/submission-repository.ts @@ -323,6 +323,18 @@ export class SubmissionRepository extends BaseRepository { return response.rows[0]; } + // TODO add return type + async getFeatureRecordsBySubmissionId(submissionId: number): Promise { + const queryBuilder = getKnex() + .select('*') + .from('submission_feature') + .where('submission_id', submissionId); + + const response = await this.connection.knex(queryBuilder); + + return response.rows; + } + /** * Get feature type id by name. * diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts new file mode 100644 index 000000000..0ac4efcb0 --- /dev/null +++ b/api/src/services/search-index-service.ts @@ -0,0 +1,25 @@ +import { IDBConnection } from "../database/db"; +import { SearchIndexRepository } from "../repositories/search-index-respository"; +import { SubmissionRepository } from "../repositories/submission-repository"; +import { getLogger } from "../utils/logger"; +import { DBService } from "./db-service"; + +const defaultLog = getLogger('services/search-index-service'); + +export class SearchIndexService extends DBService { + searchIndexRepository: SearchIndexRepository; + + constructor(connection: IDBConnection) { + super(connection); + + this.searchIndexRepository = new SearchIndexRepository(connection); + } + + async indexFeaturesBySubmissionId(submissionId: number): Promise { + const submissionService = new SubmissionRepository(this.connection); + const features = await submissionService.getFeatureRecordsBySubmissionId(submissionId); + + const featureData = features.map((feature) => feature.data); + defaultLog.debug({ label: 'indexFeaturesBySubmissionId', featureData }); + } +} diff --git a/app/src/features/home/HomePage.tsx b/app/src/features/home/HomePage.tsx index fd6773e81..da7bcaf44 100644 --- a/app/src/features/home/HomePage.tsx +++ b/app/src/features/home/HomePage.tsx @@ -2,6 +2,7 @@ import { Container, Paper, Typography } from '@mui/material'; import Box from '@mui/material/Box'; import SearchComponent from 'features/search/SearchComponent'; import { Formik, FormikProps } from 'formik'; +import { useApi } from 'hooks/useApi'; import { IAdvancedSearch } from 'interfaces/useSearchApi.interface'; import { useRef } from 'react'; import { useHistory } from 'react-router'; @@ -20,6 +21,8 @@ const HomePage = () => { history.push(`/search?keywords=${query}`); }; + useApi().submissions.test(1) + return ( { return data; }; + const test = async (submissionId: number): Promise => { + const { data } = await axios.post(`/api/dataset/search-idx?submissionId=${submissionId}`); + + return data; + } + return { listSubmissions, - getSignedUrl + getSignedUrl, + test }; }; From 35a680a479f82c3786f5eabf91ef0b6ebd552b20 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Thu, 7 Dec 2023 11:28:18 -0800 Subject: [PATCH 05/37] SIMSBIOHUB-379: Parse key-value pairs (WIP) --- .../repositories/search-index-respository.ts | 75 +++++++++++++++++-- api/src/services/search-index-service.ts | 43 ++++++++++- 2 files changed, 107 insertions(+), 11 deletions(-) diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index 5e1ebc6c8..5000e3aae 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -3,14 +3,18 @@ import { getLogger } from "../utils/logger"; import { BaseRepository } from "./base-repository"; import { getKnex } from "../database/db"; import { ApiExecuteSQLError } from "../errors/api-error"; +import SQL from "sql-template-strings"; const defaultLog = getLogger('repositories/search-index-repository'); -const Point = z.object({ +// TODO replace with pre-existing Zod types for geojson +const Geometry = z.object({ type: z.literal('Point'), coordinates: z.tuple([z.number(), z.number()]), }); +export type Geometry = z.infer; + const SearchableRecord = z.object({ submission_feature_id: z.number(), feature_property_id: z.number(), @@ -22,7 +26,7 @@ const SearchableRecord = z.object({ revision_count: z.number(), }); -type InsertSearchableRecordKey = 'value' | 'feature_property_id' +type InsertSearchableRecordKey = 'submission_feature_id' | 'value' | 'feature_property_id' export const DatetimeSearchableRecord = SearchableRecord.extend({ search_datetime_id: z.date(), @@ -36,7 +40,7 @@ export const NumberSearchableRecord = SearchableRecord.extend({ export const SpatialSearchableRecord = SearchableRecord.extend({ search_spatial_id: z.number(), - value: Point + value: Geometry }); export const StringSearchableRecord = SearchableRecord.extend({ @@ -50,13 +54,22 @@ export type NumberSearchableRecord = z.infer; export type SpatialSearchableRecord = z.infer; export type StringSearchableRecord = z.infer; -type InsertDatetimeSearchableRecord = Pick -type InsertNumberSearchableRecord = Pick -type InsertSpatialSearchableRecord = Pick -type InsertStringSearchableRecord = Pick +export type InsertDatetimeSearchableRecord = Pick +export type InsertNumberSearchableRecord = Pick +export type InsertSpatialSearchableRecord = Pick +export type InsertStringSearchableRecord = Pick +/** + * A class for creating searchable records + */ export class SearchIndexRepository extends BaseRepository { - + /** + * Inserts a searchable datetime record. + * + * @param {InsertDatetimeSearchableRecord[]} datetimeRecords + * @return {*} {Promise} + * @memberof SearchIndexRepository + */ async insertSearchableDatetimeRecords(datetimeRecords: InsertDatetimeSearchableRecord[]): Promise { defaultLog.debug({ label: 'insertSearchableDatetimeRecords' }); @@ -78,6 +91,13 @@ export class SearchIndexRepository extends BaseRepository { return response.rows; } + /** + * Inserts a searchable number record. + * + * @param {InsertNumberSearchableRecord[]} numberRecords + * @return {*} {Promise} + * @memberof SearchIndexRepository + */ async insertSearchableNumberRecords(numberRecords: InsertNumberSearchableRecord[]): Promise { defaultLog.debug({ label: 'insertSearchableNumberRecords' }); @@ -99,6 +119,13 @@ export class SearchIndexRepository extends BaseRepository { return response.rows; } + /** + * Inserts a searchable spatial record. + * + * @param {InsertSpatialSearchableRecord[]} spatialRecords + * @return {*} {Promise} + * @memberof SearchIndexRepository + */ async insertSearchableSpatialRecords(spatialRecords: InsertSpatialSearchableRecord[]): Promise { defaultLog.debug({ label: 'insertSearchableSpatialRecords' }); @@ -120,6 +147,13 @@ export class SearchIndexRepository extends BaseRepository { return response.rows; } + /** + * Inserts a searchable string record. + * + * @param {InsertStringSearchableRecord[]} stringRecords + * @return {*} {Promise} + * @memberof SearchIndexRepository + */ async insertSearchableStringRecords(stringRecords: InsertStringSearchableRecord[]): Promise { defaultLog.debug({ label: 'insertSearchableStringRecords' }); @@ -140,4 +174,29 @@ export class SearchIndexRepository extends BaseRepository { return response.rows; } + + // TODO return type + async getFeaturePropertiesWithTypeNames(): Promise { + const query = SQL` + SELECT + fp.name as property_name, + fpt.name as property_type, + fp.* + FROM + feature_property fp + LEFT JOIN + feature_property_type fpt + ON + fp.feature_property_type_id = fpt.feature_property_type_id + WHERE + fp.record_end_date IS NULL + AND + fpt.record_end_date IS NULL + `; + + const response = await this.connection.sql(query); + + return response.rows; + } + } diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 0ac4efcb0..b1339c252 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -1,5 +1,5 @@ import { IDBConnection } from "../database/db"; -import { SearchIndexRepository } from "../repositories/search-index-respository"; +import { Geometry, InsertDatetimeSearchableRecord, InsertNumberSearchableRecord, InsertSpatialSearchableRecord, InsertStringSearchableRecord, SearchIndexRepository } from "../repositories/search-index-respository"; import { SubmissionRepository } from "../repositories/submission-repository"; import { getLogger } from "../utils/logger"; import { DBService } from "./db-service"; @@ -19,7 +19,44 @@ export class SearchIndexService extends DBService { const submissionService = new SubmissionRepository(this.connection); const features = await submissionService.getFeatureRecordsBySubmissionId(submissionId); - const featureData = features.map((feature) => feature.data); - defaultLog.debug({ label: 'indexFeaturesBySubmissionId', featureData }); + const datetimeRecords: InsertDatetimeSearchableRecord[] = []; + const numberRecords: InsertNumberSearchableRecord[] = []; + const spatialRecords: InsertSpatialSearchableRecord[] = []; + const stringRecords: InsertStringSearchableRecord[] = []; + + const featurePropertyTypeNames = await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); + + const propertyTypeMap = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { + const { property_name, ...rest } = propertyType; + return [property_name, rest]; + })) + + features.forEach((feature) => { + const { submission_feature_id } = feature; + Object + .entries(feature.data.properties) + .forEach(([property_name, value]) => { + const { property_type, feature_property_id } = propertyTypeMap[property_name]; + switch (property_type) { + case 'datetime': + datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); + break; + + case 'number': + numberRecords.push({ submission_feature_id, feature_property_id, value: value as number }); + break; + + case 'spatial': + spatialRecords.push({ submission_feature_id, feature_property_id, value: value as Geometry }); + break; + + case 'string': + stringRecords.push({ submission_feature_id, feature_property_id, value: value as string }); + break; + } + }) + }); + + defaultLog.debug({ label: 'indexFeaturesBySubmissionId', datetimeRecords, numberRecords, spatialRecords, stringRecords }); } } From 85a2aa4694a1f2771bc563306d418ed891989dc3 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Fri, 8 Dec 2023 14:34:15 -0800 Subject: [PATCH 06/37] SIMSBIOHUB-379: fix typo --- api/src/services/search-index-service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index b1339c252..251afd1f6 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -16,8 +16,8 @@ export class SearchIndexService extends DBService { } async indexFeaturesBySubmissionId(submissionId: number): Promise { - const submissionService = new SubmissionRepository(this.connection); - const features = await submissionService.getFeatureRecordsBySubmissionId(submissionId); + const submissionRepository = new SubmissionRepository(this.connection); + const features = await submissionRepository.getFeatureRecordsBySubmissionId(submissionId); const datetimeRecords: InsertDatetimeSearchableRecord[] = []; const numberRecords: InsertNumberSearchableRecord[] = []; From eb023bdc863c6fe9a7bc035cfd86df16e7dfc11f Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Fri, 8 Dec 2023 15:21:58 -0800 Subject: [PATCH 07/37] Merge branch 'dataset_security_feature' into SIMSBIOHUB-379 --- api/src/database/db.ts | 13 + api/src/openapi/schemas/http-responses.ts | 2 +- api/src/paths/administrative/review/list.ts | 103 ------- .../unreviewed/index.test.ts} | 23 +- .../submission/unreviewed/index.ts | 122 +++++++++ api/src/paths/dataset/{datasetId}/index.ts | 77 ++++++ api/src/repositories/artifact-repository.ts | 4 +- .../submission-repository.test.ts | 48 ++++ api/src/repositories/submission-repository.ts | 70 ++++- api/src/repositories/user-repository.ts | 8 +- api/src/services/dataset-service.ts | 27 ++ api/src/services/submission-service.test.ts | 47 ++++ api/src/services/submission-service.ts | 11 + api/src/services/validation-service.ts | 6 +- app/package-lock.json | 5 + app/package.json | 1 + app/src/AppRouter.tsx | 7 + .../components/layout/header/Header.test.tsx | 2 +- app/src/components/layout/header/Header.tsx | 9 +- .../components/security/ManageSecurity.tsx | 49 ++++ .../components/security/UnsecureDialog.tsx | 36 +++ .../admin/dashboard/DashboardPage.test.tsx | 6 +- .../admin/dashboard/DashboardPage.tsx | 2 + .../components/DatasetsForReviewTable.tsx | 71 ++--- app/src/features/datasets/DatasetPage.tsx | 256 ++++++++---------- app/src/features/datasets/DatasetsRouter.tsx | 6 +- app/src/features/search/SearchComponent.tsx | 48 +++- .../submissions/SubmissionsListPage.tsx | 147 ++++++++++ .../submissions/SubmissionsRouter.tsx | 28 ++ .../components/SubmissionsListSortMenu.tsx | 96 +++++++ app/src/hooks/api/useDatasetApi.ts | 28 +- app/src/hooks/api/useSubmissionsApi.ts | 48 +++- app/src/hooks/useDebounce.test.tsx | 21 ++ app/src/hooks/useDebounce.tsx | 29 ++ app/src/hooks/useDownloadJSON.tsx | 25 ++ app/src/hooks/useFuzzySearch.test.tsx | 102 +++++++ app/src/hooks/useFuzzySearch.tsx | 144 ++++++++++ app/src/interfaces/useDatasetApi.interface.ts | 9 + .../interfaces/useSubmissionsApi.interface.ts | 13 + database/package-lock.json | 54 ---- .../20231117000001_security_tables.ts | 89 ------ .../src/migrations/release.0.8.0/biohub.sql | 78 +----- .../tr_generated_audit_triggers.sql | 1 - .../tr_generated_journal_triggers.sql | 1 - .../src/seeds/02_populate_feature_tables.ts | 7 +- database/src/seeds/04_mock_test_data.ts | 32 ++- docker-compose.yml | 2 + 47 files changed, 1453 insertions(+), 560 deletions(-) delete mode 100644 api/src/paths/administrative/review/list.ts rename api/src/paths/administrative/{review/list.test.ts => submission/unreviewed/index.test.ts} (63%) create mode 100644 api/src/paths/administrative/submission/unreviewed/index.ts create mode 100644 api/src/paths/dataset/{datasetId}/index.ts create mode 100644 api/src/services/dataset-service.ts create mode 100644 app/src/components/security/ManageSecurity.tsx create mode 100644 app/src/components/security/UnsecureDialog.tsx create mode 100644 app/src/features/submissions/SubmissionsListPage.tsx create mode 100644 app/src/features/submissions/SubmissionsRouter.tsx create mode 100644 app/src/features/submissions/components/SubmissionsListSortMenu.tsx create mode 100644 app/src/hooks/useDebounce.test.tsx create mode 100644 app/src/hooks/useDebounce.tsx create mode 100644 app/src/hooks/useDownloadJSON.tsx create mode 100644 app/src/hooks/useFuzzySearch.test.tsx create mode 100644 app/src/hooks/useFuzzySearch.tsx diff --git a/api/src/database/db.ts b/api/src/database/db.ts index 680227de4..1a1e923c9 100644 --- a/api/src/database/db.ts +++ b/api/src/database/db.ts @@ -42,6 +42,19 @@ export const defaultPoolConfig: pg.PoolConfig = { pg.types.setTypeParser(pg.types.builtins.DATE, (stringValue: string) => { return stringValue; // 1082 for `DATE` type }); +// Adding a TIMESTAMP type parser to keep all dates used in the system consistent +pg.types.setTypeParser(pg.types.builtins.TIMESTAMP, (stringValue: string) => { + return stringValue; // 1082 for `TIMESTAMP` type +}); +// Adding a TIMESTAMPTZ type parser to keep all dates used in the system consistent +pg.types.setTypeParser(pg.types.builtins.TIMESTAMPTZ, (stringValue: string) => { + return stringValue; // 1082 for `DATE` type +}); +// NUMERIC column types return as strings to maintain precision. Converting this to a float so it is usable by the system +// Explanation of why Numeric returns as a string: https://github.com/brianc/node-postgres/issues/811 +pg.types.setTypeParser(pg.types.builtins.NUMERIC, (stringValue: string) => { + return parseFloat(stringValue); +}); // singleton pg pool instance used by the api let DBPool: pg.Pool | undefined; diff --git a/api/src/openapi/schemas/http-responses.ts b/api/src/openapi/schemas/http-responses.ts index 024aff518..45e47b4cd 100644 --- a/api/src/openapi/schemas/http-responses.ts +++ b/api/src/openapi/schemas/http-responses.ts @@ -6,7 +6,7 @@ export const defaultErrorResponses = { $ref: '#/components/responses/401' }, 403: { - $ref: '#/components/responses/401' + $ref: '#/components/responses/403' }, 409: { $ref: '#/components/responses/409' diff --git a/api/src/paths/administrative/review/list.ts b/api/src/paths/administrative/review/list.ts deleted file mode 100644 index 1fe0aa44b..000000000 --- a/api/src/paths/administrative/review/list.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { RequestHandler } from 'express'; -import { Operation } from 'express-openapi'; -import { SYSTEM_ROLE } from '../../../constants/roles'; -import { getDBConnection } from '../../../database/db'; -import { defaultErrorResponses } from '../../../openapi/schemas/http-responses'; -import { authorizeRequestHandler } from '../../../request-handlers/security/authorization'; -import { SubmissionService } from '../../../services/submission-service'; -import { getLogger } from '../../../utils/logger'; - -const defaultLog = getLogger('paths/administrative/review/list'); - -export const GET: Operation = [ - authorizeRequestHandler(() => { - return { - and: [ - { - validSystemRoles: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.DATA_ADMINISTRATOR], - discriminator: 'SystemRole' - } - ] - }; - }), - getDatasetsForReview() -]; - -GET.apiDoc = { - description: 'Get a list of datasets that need review.', - tags: ['admin'], - security: [ - { - Bearer: [] - } - ], - parameters: [], - responses: { - 200: { - description: 'List of datasets that need review.', - content: { - 'application/json': { - schema: { - type: 'array', - items: { - type: 'object', - properties: { - dataset_id: { - type: 'string', - description: 'UUID for a specific dataset' - }, - artifacts_to_review: { - type: 'integer', - description: 'A count of the total files to review' - }, - dataset_name: { - type: 'string', - description: 'Name of the project to review' - }, - last_updated: { - type: 'string', - description: 'Last date a file was updated' - }, - keywords: { - type: 'array', - items: { - type: 'string', - description: 'Keyword used for filtering datasets' - } - } - } - } - } - } - } - }, - ...defaultErrorResponses - } -}; - -/** - * Get all administrative activities for the specified type, or all if no type is provided. - * - * @returns {RequestHandler} - */ -export function getDatasetsForReview(): RequestHandler { - return async (req, res) => { - const connection = getDBConnection(req['keycloak_token']); - - try { - await connection.open(); - - await connection.commit(); - - const service = new SubmissionService(connection); - const response = await service.getDatasetsForReview(['PROJECT']); - - return res.status(200).json(response); - } catch (error) { - defaultLog.error({ label: 'getDatasetsForReview', message: 'error', error }); - throw error; - } finally { - connection.release(); - } - }; -} diff --git a/api/src/paths/administrative/review/list.test.ts b/api/src/paths/administrative/submission/unreviewed/index.test.ts similarity index 63% rename from api/src/paths/administrative/review/list.test.ts rename to api/src/paths/administrative/submission/unreviewed/index.test.ts index 705024e64..4fdfcdb48 100644 --- a/api/src/paths/administrative/review/list.test.ts +++ b/api/src/paths/administrative/submission/unreviewed/index.test.ts @@ -2,11 +2,11 @@ import chai, { expect } from 'chai'; import { describe } from 'mocha'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import * as db from '../../../database/db'; -import { HTTPError } from '../../../errors/http-error'; -import { SubmissionService } from '../../../services/submission-service'; -import { getMockDBConnection, getRequestHandlerMocks } from '../../../__mocks__/db'; -import * as list from './list'; +import { getUnreviewedSubmissions } from '.'; +import * as db from '../../../../database/db'; +import { HTTPError } from '../../../../errors/http-error'; +import { SubmissionService } from '../../../../services/submission-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../__mocks__/db'; chai.use(sinonChai); @@ -26,7 +26,7 @@ describe('list', () => { const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); - const requestHandler = list.getDatasetsForReview(); + const requestHandler = getUnreviewedSubmissions(); try { await requestHandler(mockReq, mockRes, mockNext); @@ -36,7 +36,7 @@ describe('list', () => { } }); - it('should return 200 after update is completed', async () => { + it('should return an array of unreviewed submission objects', async () => { const dbConnectionObj = getMockDBConnection({ commit: sinon.stub(), rollback: sinon.stub(), @@ -47,13 +47,16 @@ describe('list', () => { const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); - const mock = sinon.stub(SubmissionService.prototype, 'getDatasetsForReview').resolves(); + const getUnreviewedSubmissionsStub = sinon + .stub(SubmissionService.prototype, 'getUnreviewedSubmissions') + .resolves([]); - const requestHandler = list.getDatasetsForReview(); + const requestHandler = getUnreviewedSubmissions(); await requestHandler(mockReq, mockRes, mockNext); - expect(mock).to.have.been.calledOnce; + expect(getUnreviewedSubmissionsStub).to.have.been.calledOnce; expect(mockRes.statusValue).to.equal(200); + expect(mockRes.jsonValue).to.eql([]); }); }); diff --git a/api/src/paths/administrative/submission/unreviewed/index.ts b/api/src/paths/administrative/submission/unreviewed/index.ts new file mode 100644 index 000000000..830eb8c18 --- /dev/null +++ b/api/src/paths/administrative/submission/unreviewed/index.ts @@ -0,0 +1,122 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { SYSTEM_ROLE } from '../../../../constants/roles'; +import { getDBConnection } from '../../../../database/db'; +import { defaultErrorResponses } from '../../../../openapi/schemas/http-responses'; +import { authorizeRequestHandler } from '../../../../request-handlers/security/authorization'; +import { SubmissionService } from '../../../../services/submission-service'; +import { getLogger } from '../../../../utils/logger'; + +const defaultLog = getLogger('paths/administrative/submission/unreviewed'); + +export const GET: Operation = [ + authorizeRequestHandler(() => { + return { + and: [ + { + validSystemRoles: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + getUnreviewedSubmissions() +]; + +GET.apiDoc = { + description: 'Get a list of submissions that need security review (are unreviewed).', + tags: ['admin'], + security: [ + { + Bearer: [] + } + ], + responses: { + 200: { + description: 'List of submissions that need security review.', + content: { + 'application/json': { + schema: { + type: 'array', + items: { + type: 'object', + properties: { + submission_id: { + type: 'integer', + minimum: 1 + }, + uuid: { + type: 'string', + format: 'uuid' + }, + security_review_timestamp: { + type: 'string', + nullable: true + }, + source_system: { + type: 'string' + }, + name: { + type: 'string', + maxLength: 200 + }, + description: { + type: 'string', + maxLength: 3000 + }, + create_date: { + type: 'string' + }, + create_user: { + type: 'integer', + minimum: 1 + }, + update_date: { + type: 'string', + nullable: true + }, + update_user: { + type: 'integer', + minimum: 1, + nullable: true + }, + revision_count: { + type: 'integer', + minimum: 0 + } + } + } + } + } + } + }, + ...defaultErrorResponses + } +}; + +/** + * Get all unreviewed submissions. + * + * @returns {RequestHandler} + */ +export function getUnreviewedSubmissions(): RequestHandler { + return async (req, res) => { + const connection = getDBConnection(req['keycloak_token']); + + try { + await connection.open(); + + await connection.commit(); + + const service = new SubmissionService(connection); + const response = await service.getUnreviewedSubmissions(); + + return res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'getUnreviewedSubmissions', message: 'error', error }); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/paths/dataset/{datasetId}/index.ts b/api/src/paths/dataset/{datasetId}/index.ts new file mode 100644 index 000000000..81fe23890 --- /dev/null +++ b/api/src/paths/dataset/{datasetId}/index.ts @@ -0,0 +1,77 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { getAPIUserDBConnection, getDBConnection } from '../../../database/db'; +import { defaultErrorResponses } from '../../../openapi/schemas/http-responses'; +import { DatasetService } from '../../../services/dataset-service'; +import { getLogger } from '../../../utils/logger'; + +const defaultLog = getLogger('paths/dataset/{datasetId}'); + +export const GET: Operation = [getDatasetInformation()]; + +GET.apiDoc = { + description: 'retrieves dataset data from the submission table', + tags: ['eml'], + security: [ + { + OptionalBearer: [] + } + ], + parameters: [ + { + description: 'dataset uuid', + in: 'path', + name: 'datasetId', + schema: { + type: 'string', + format: 'uuid' + }, + required: true + } + ], + responses: { + 200: { + description: 'Dataset metadata response object.', + content: { + 'application/json': { + schema: { + type: 'object' + //TODO: add schema + } + } + } + }, + ...defaultErrorResponses + } +}; + +/** + * Retrieves dataset data from the submission table. + * + * @returns {RequestHandler} + */ +export function getDatasetInformation(): RequestHandler { + return async (req, res) => { + const connection = req['keycloak_token'] ? getDBConnection(req['keycloak_token']) : getAPIUserDBConnection(); + + const datasetId = String(req.params.datasetId); + + try { + await connection.open(); + + const datasetService = new DatasetService(connection); + + const result = await datasetService.getDatasetByDatasetUUID(datasetId); + + await connection.commit(); + + res.status(200).json(result); + } catch (error) { + defaultLog.error({ label: 'getMetadataByDatasetId', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/repositories/artifact-repository.ts b/api/src/repositories/artifact-repository.ts index 370c0a2bf..1a7212b1f 100644 --- a/api/src/repositories/artifact-repository.ts +++ b/api/src/repositories/artifact-repository.ts @@ -23,8 +23,8 @@ export const Artifact = ArtifactMetadata.extend({ uuid: z.string().uuid(), key: z.string(), foi_reason: z.boolean().nullable().optional(), - security_review_timestamp: z.date().nullable().optional(), - create_date: z.date().optional() + security_review_timestamp: z.string().nullable().optional(), + create_date: z.string().optional() }); export type Artifact = z.infer; diff --git a/api/src/repositories/submission-repository.test.ts b/api/src/repositories/submission-repository.test.ts index f994a4f5b..7cc5d4d64 100644 --- a/api/src/repositories/submission-repository.test.ts +++ b/api/src/repositories/submission-repository.test.ts @@ -9,6 +9,7 @@ import { getMockDBConnection } from '../__mocks__/db'; import { ISourceTransformModel, ISpatialComponentCount, + SubmissionRecord, SubmissionRepository, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE @@ -896,4 +897,51 @@ describe('SubmissionRepository', () => { expect(response).to.eql(mockResponse); }); }); + + describe('getUnreviewedSubmissions', () => { + beforeEach(() => { + sinon.restore(); + }); + + it('should succeed with valid data', async () => { + const mockSubmissionRecords: SubmissionRecord[] = [ + { + submission_id: 1, + uuid: '123-456-789', + security_review_timestamp: null, + source_system: 'SIMS', + name: 'name', + description: 'description', + create_date: '2023-12-12', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + submission_id: 2, + uuid: '789-456-123', + security_review_timestamp: '2023-12-12', + source_system: 'SIMS', + name: 'name', + description: 'description', + create_date: '2023-12-12', + create_user: 1, + update_date: '2023-12-12', + update_user: 1, + revision_count: 1 + } + ]; + + const mockResponse = { rowCount: 2, rows: mockSubmissionRecords } as unknown as Promise>; + + const mockDBConnection = getMockDBConnection({ sql: async () => mockResponse }); + + const submissionRepository = new SubmissionRepository(mockDBConnection); + + const response = await submissionRepository.getUnreviewedSubmissions(); + + expect(response).to.eql(mockSubmissionRecords); + }); + }); }); diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts index ed6a9c6cb..0b7953d4c 100644 --- a/api/src/repositories/submission-repository.ts +++ b/api/src/repositories/submission-repository.ts @@ -90,8 +90,8 @@ export interface ISubmissionRecordWithSpatial { */ export interface ISubmissionModel { submission_id?: number; - source_transform_id: number; uuid: string; + security_review_timestamp?: string | null; create_date?: string; create_user?: number; update_date?: string | null; @@ -204,6 +204,22 @@ export interface ISubmissionObservationRecord { revision_count?: string; } +export const SubmissionRecord = z.object({ + submission_id: z.number(), + uuid: z.string(), + security_review_timestamp: z.string().nullable(), + source_system: z.string(), + name: z.string(), + description: z.string().nullable(), + create_date: z.string(), + create_user: z.number(), + update_date: z.string().nullable(), + update_user: z.number().nullable(), + revision_count: z.number() +}); + +export type SubmissionRecord = z.infer; + /** * A repository class for accessing submission data. * @@ -1082,4 +1098,56 @@ export class SubmissionRepository extends BaseRepository { return response.rows[0]; } + + /** + * Fetch a submission from uuid. + * + * @param {string} uuid + * @return {*} {Promise} + * @memberof SubmissionRepository + */ + async getSubmissionByUUID(uuid: string): Promise { + const sqlStatement = SQL` + SELECT + submission_id, + uuid, + security_review_timestamp + FROM + submission + WHERE + uuid = ${uuid}; + `; + + const response = await this.connection.sql(sqlStatement); + + if (!response.rowCount) { + throw new ApiExecuteSQLError('Failed to get submission record', [ + 'SubmissionRepository->getSubmissionByUUID', + 'rowCount was null or undefined, expected rowCount != 0' + ]); + } + + return response.rows[0]; + } + + /** + * Get all submissions that are pending security review (are unreviewed). + * + * @return {*} {Promise} + * @memberof SubmissionRepository + */ + async getUnreviewedSubmissions(): Promise { + const sqlStatement = SQL` + SELECT + * + FROM + submission + WHERE + submission.security_review_timestamp is null; + `; + + const response = await this.connection.sql(sqlStatement, SubmissionRecord); + + return response.rows; + } } diff --git a/api/src/repositories/user-repository.ts b/api/src/repositories/user-repository.ts index 8569b9f8a..08e90617a 100644 --- a/api/src/repositories/user-repository.ts +++ b/api/src/repositories/user-repository.ts @@ -9,11 +9,11 @@ export const SystemUser = z.object({ user_identity_source_id: z.number(), user_identifier: z.string(), user_guid: z.string(), - record_effective_date: z.date(), - record_end_date: z.date().nullable(), - create_date: z.date(), + record_effective_date: z.string(), + record_end_date: z.string().nullable(), + create_date: z.string(), create_user: z.number(), - update_date: z.date().nullable(), + update_date: z.string().nullable(), update_user: z.number().nullable(), revision_count: z.number() }); diff --git a/api/src/services/dataset-service.ts b/api/src/services/dataset-service.ts new file mode 100644 index 000000000..d15ccaa13 --- /dev/null +++ b/api/src/services/dataset-service.ts @@ -0,0 +1,27 @@ +import { IDBConnection } from '../database/db'; +import { SubmissionRepository } from '../repositories/submission-repository'; +import { DBService } from './db-service'; + +export class DatasetService extends DBService { + submissionRepository: SubmissionRepository; + + constructor(connection: IDBConnection) { + super(connection); + + this.submissionRepository = new SubmissionRepository(connection); + } + + /** + * Retrieves dataset data from the submission table. + * + * @param {string} datasetUUID + * @return {*} {Promise} + * @memberof DatasetService + */ + async getDatasetByDatasetUUID(datasetUUID: string): Promise { + const submission = this.submissionRepository.getSubmissionByUUID(datasetUUID); + console.log('submission', submission); + + return submission; + } +} diff --git a/api/src/services/submission-service.test.ts b/api/src/services/submission-service.test.ts index 26b63d33c..244f9fcaa 100644 --- a/api/src/services/submission-service.test.ts +++ b/api/src/services/submission-service.test.ts @@ -11,6 +11,7 @@ import { ISubmissionJobQueueRecord, ISubmissionModel, ISubmissionObservationRecord, + SubmissionRecord, SubmissionRepository, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE @@ -703,4 +704,50 @@ describe('SubmissionService', () => { }); }); }); + + describe('getUnreviewedSubmissions', () => { + it('should return an array of submission records', async () => { + const mockSubmissionRecords: SubmissionRecord[] = [ + { + submission_id: 1, + uuid: '123-456-789', + security_review_timestamp: null, + source_system: 'SIMS', + name: 'name', + description: 'description', + create_date: '2023-12-12', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + submission_id: 2, + uuid: '789-456-123', + security_review_timestamp: '2023-12-12', + source_system: 'SIMS', + name: 'name', + description: 'description', + create_date: '2023-12-12', + create_user: 1, + update_date: '2023-12-12', + update_user: 1, + revision_count: 1 + } + ]; + + const mockDBConnection = getMockDBConnection(); + + const getUnreviewedSubmissionsStub = sinon + .stub(SubmissionRepository.prototype, 'getUnreviewedSubmissions') + .resolves(mockSubmissionRecords); + + const submissionService = new SubmissionService(mockDBConnection); + + const response = await submissionService.getUnreviewedSubmissions(); + + expect(getUnreviewedSubmissionsStub).to.be.calledOnce; + expect(response).to.be.eql(mockSubmissionRecords); + }); + }); }); diff --git a/api/src/services/submission-service.ts b/api/src/services/submission-service.ts index 55cd4434f..c75add3a5 100644 --- a/api/src/services/submission-service.ts +++ b/api/src/services/submission-service.ts @@ -15,6 +15,7 @@ import { ISubmissionObservationRecord, ISubmissionRecord, ISubmissionRecordWithSpatial, + SubmissionRecord, SubmissionRepository, SUBMISSION_MESSAGE_TYPE, SUBMISSION_STATUS_TYPE @@ -533,4 +534,14 @@ export class SubmissionService extends DBService { async updateSubmissionMetadataWithSearchKeys(submissionId: number, datasetSearch: any): Promise { return this.submissionRepository.updateSubmissionMetadataWithSearchKeys(submissionId, datasetSearch); } + + /** + * Get all submissions that are pending security review (are unreviewed). + * + * @return {*} {Promise} + * @memberof SubmissionService + */ + async getUnreviewedSubmissions(): Promise { + return this.submissionRepository.getUnreviewedSubmissions(); + } } diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 9d79c1de5..07189a64b 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -10,7 +10,7 @@ import { ICsvState } from '../utils/media/csv/csv-file'; import { DWCArchive } from '../utils/media/dwc/dwc-archive-file'; import { IMediaState } from '../utils/media/media-file'; import { ValidationSchemaParser } from '../utils/media/validation/validation-schema-parser'; -import { GeoJSONFeatureZodSchema } from '../zod-schema/geoJsonZodSchema'; +import { GeoJSONFeatureCollectionZodSchema } from '../zod-schema/geoJsonZodSchema'; import { DBService } from './db-service'; export class ValidationService extends DBService { @@ -63,6 +63,8 @@ export class ValidationService extends DBService { } validateProperties(properties: IFeatureProperties[], dataProperties: any): boolean { + console.log('dataProperties', dataProperties); + console.log('properties', properties); const throwPropertyError = (property: IFeatureProperties) => { throw new Error(`Property ${property.name} is not of type ${property.type}`); }; @@ -96,7 +98,7 @@ export class ValidationService extends DBService { } break; case 'spatial': { - const { success } = GeoJSONFeatureZodSchema.safeParse(dataProperty); + const { success } = GeoJSONFeatureCollectionZodSchema.safeParse(dataProperty); if (!success) { throwPropertyError(property); } diff --git a/app/package-lock.json b/app/package-lock.json index 5179e084e..68083ca84 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -9220,6 +9220,11 @@ "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==" }, + "fuse.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.0.0.tgz", + "integrity": "sha512-14F4hBIxqKvD4Zz/XjDc3y94mNZN6pRv3U13Udo0lNLCWRBUsrMv2xwcF/y/Z5sV6+FQW+/ow68cHpm4sunt8Q==" + }, "gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", diff --git a/app/package.json b/app/package.json index a11c2e0c3..4152e9fb5 100644 --- a/app/package.json +++ b/app/package.json @@ -50,6 +50,7 @@ "dompurify": "^2.4.0", "express": "~4.17.1", "formik": "~2.2.6", + "fuse.js": "^7.0.0", "handlebars": "^4.7.7", "jest-watch-typeahead": "^2.2.2", "jszip": "^3.10.1", diff --git a/app/src/AppRouter.tsx b/app/src/AppRouter.tsx index 91ee869b8..454ea37e0 100644 --- a/app/src/AppRouter.tsx +++ b/app/src/AppRouter.tsx @@ -9,6 +9,7 @@ import DatasetsRouter from 'features/datasets/DatasetsRouter'; import HomeRouter from 'features/home/HomeRouter'; import MapRouter from 'features/map/MapRouter'; import SearchRouter from 'features/search/SearchRouter'; +import SubmissionsRouter from 'features/submissions/SubmissionsRouter'; import BaseLayout from 'layouts/BaseLayout'; import ContentLayout from 'layouts/ContentLayout'; import LoginPage from 'pages/authentication/LoginPage'; @@ -42,6 +43,12 @@ const AppRouter: React.FC = () => { + + + + + + diff --git a/app/src/components/layout/header/Header.test.tsx b/app/src/components/layout/header/Header.test.tsx index 66502051f..0eb3d6f51 100644 --- a/app/src/components/layout/header/Header.test.tsx +++ b/app/src/components/layout/header/Header.test.tsx @@ -25,7 +25,7 @@ describe('Header', () => { ); expect(getByText('Home')).toBeVisible(); - expect(getByText('Find Datasets')).toBeVisible(); + expect(getByText('Submissions')).toBeVisible(); expect(getByText('Map Search')).toBeVisible(); expect(getByText('Manage Users')).toBeVisible(); }); diff --git a/app/src/components/layout/header/Header.tsx b/app/src/components/layout/header/Header.tsx index 4ba230e16..4ba90a7d4 100644 --- a/app/src/components/layout/header/Header.tsx +++ b/app/src/components/layout/header/Header.tsx @@ -139,8 +139,8 @@ const Header: React.FC = () => { Dashboard - - Find Datasets + + Submissions Map Search @@ -150,6 +150,11 @@ const Header: React.FC = () => { Manage Users + + + Submission 1 + + diff --git a/app/src/components/security/ManageSecurity.tsx b/app/src/components/security/ManageSecurity.tsx new file mode 100644 index 000000000..c29ea5a9c --- /dev/null +++ b/app/src/components/security/ManageSecurity.tsx @@ -0,0 +1,49 @@ +import { mdiChevronDown, mdiChevronUp, mdiLock, mdiLockOpenVariantOutline } from '@mdi/js'; +import Icon from '@mdi/react'; +import { Button, Menu, MenuItem } from '@mui/material'; +import ListItemIcon from '@mui/material/ListItemIcon'; +import ListItemText from '@mui/material/ListItemText'; +import React, { useState } from 'react'; +import UnsecureDialog from './UnsecureDialog'; + +const ManageSecurity = () => { + const [anchorEl, setAnchorEl] = React.useState(null); + const [isUnsecureDialogOpen, setIsUnsecuredDialogOpen] = useState(false); + const open = Boolean(anchorEl); + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + const handleClose = () => { + setAnchorEl(null); + }; + + return ( + <> + setIsUnsecuredDialogOpen(false)} /> + + + + + + + Secure Records + + setIsUnsecuredDialogOpen(true)}> + + + + Unsecure Records + + + + ); +}; + +export default ManageSecurity; diff --git a/app/src/components/security/UnsecureDialog.tsx b/app/src/components/security/UnsecureDialog.tsx new file mode 100644 index 000000000..4cf50d6bd --- /dev/null +++ b/app/src/components/security/UnsecureDialog.tsx @@ -0,0 +1,36 @@ +import { Alert, AlertTitle, Typography } from '@mui/material'; +import YesNoDialog from 'components/dialog/YesNoDialog'; + +interface IUnsecureDialogProps { + isOpen: boolean; + onClose: () => void; +} +const UnsecureDialog = (props: IUnsecureDialogProps) => { + return ( + { + console.log('Unsecure these fools'); + }} + onNo={props.onClose} + onClose={() => {}} + dialogTitle="Unsecure Records?" + dialogContent={ + + + Open access to all records + + + Users will be able to access and download all records included in this dataset + + + } + dialogText="Are you sure you want to unsecure this dataset?" + yesButtonProps={{ color: 'error' }} + yesButtonLabel="UNSECURE" + noButtonLabel="CANCEL" + /> + ); +}; + +export default UnsecureDialog; diff --git a/app/src/features/admin/dashboard/DashboardPage.test.tsx b/app/src/features/admin/dashboard/DashboardPage.test.tsx index 13d85b8a3..c5d8ce59c 100644 --- a/app/src/features/admin/dashboard/DashboardPage.test.tsx +++ b/app/src/features/admin/dashboard/DashboardPage.test.tsx @@ -18,7 +18,7 @@ const mockUseKeycloakWrapper = { const mockUseApi = { dataset: { - listAllDatasetsForReview: jest.fn() + getUnreviewedSubmissions: jest.fn() } }; @@ -46,7 +46,7 @@ describe('DashboardPage', () => { }); it('renders a page with no security reviews', async () => { - mockUseApi.dataset.listAllDatasetsForReview.mockResolvedValue([]); + mockUseApi.dataset.getUnreviewedSubmissions.mockResolvedValue([]); const { getByTestId } = renderContainer(); @@ -56,7 +56,7 @@ describe('DashboardPage', () => { }); it('renders a page with a table of security reviews', async () => { - mockUseApi.dataset.listAllDatasetsForReview.mockResolvedValue([ + mockUseApi.dataset.getUnreviewedSubmissions.mockResolvedValue([ { dataset_id: 'UUID-1', artifacts_to_review: 6, diff --git a/app/src/features/admin/dashboard/DashboardPage.tsx b/app/src/features/admin/dashboard/DashboardPage.tsx index 8e78f54a0..ca4eb5b84 100644 --- a/app/src/features/admin/dashboard/DashboardPage.tsx +++ b/app/src/features/admin/dashboard/DashboardPage.tsx @@ -1,6 +1,7 @@ import { Typography } from '@mui/material'; import Box from '@mui/material/Box'; import Paper from '@mui/material/Paper'; +import ManageSecurity from 'components/security/ManageSecurity'; import DatasetsForReviewTable from './components/DatasetsForReviewTable'; const DashboardPage = () => { @@ -31,6 +32,7 @@ const DashboardPage = () => { }}> Pending Security Reviews + = () => { const biohubApi = useApi(); - const unsecuredDatasetDataLoader = useDataLoader(() => biohubApi.dataset.listAllDatasetsForReview()); + + const unsecuredDatasetDataLoader = useDataLoader(() => biohubApi.dataset.getUnreviewedSubmissions()); unsecuredDatasetDataLoader.load(); - const datasetList: IDatasetForReview[] = unsecuredDatasetDataLoader.data ?? []; - const columns: GridColDef[] = [ + const datasetList: IUnreviewedSubmission[] = unsecuredDatasetDataLoader.data ?? []; + const columns: GridColDef[] = [ + { + field: 'submission_id', + headerName: 'ID', + flex: 1, + disableColumnMenu: true + }, { - field: 'artifacts_to_review', - headerName: 'FILES TO REVIEW', + field: 'uuid', + headerName: 'UUID', flex: 1, disableColumnMenu: true }, { - field: 'dataset_name', - headerName: 'TITLE', + field: 'name', + headerName: 'Name', flex: 2, - disableColumnMenu: true, - renderCell: (params: GridRenderCellParams) => { - return ( - - {params.row.dataset_name} - - ); - } + disableColumnMenu: true }, { - field: 'dataset_type', - headerName: 'TYPE', - flex: 1, - disableColumnMenu: true, - renderCell: (params: GridRenderCellParams) => { - return params.row.keywords.map((item) => ( - - )); - } + field: 'description', + headerName: 'Description', + flex: 2, + disableColumnMenu: true }, { - field: 'last_updated', - headerName: 'LAST UPDATED', + field: 'create_date', + headerName: 'Date Created', flex: 1, disableColumnMenu: true } ]; - const prepKeyword = (keyword: string): string => { - let prep = keyword.toUpperCase(); - if (prep === 'PROJECT') { - prep = 'INVENTORY PROJECT'; - } - return prep; - }; - return ( <> {unsecuredDatasetDataLoader.isLoading && } @@ -96,7 +75,7 @@ const DatasetsForReviewTable: React.FC = () => { row.dataset_id} + getRowId={(row) => row.submission_id} autoHeight rows={datasetList} columns={columns} diff --git a/app/src/features/datasets/DatasetPage.tsx b/app/src/features/datasets/DatasetPage.tsx index 800136dce..d2d2b605d 100644 --- a/app/src/features/datasets/DatasetPage.tsx +++ b/app/src/features/datasets/DatasetPage.tsx @@ -1,28 +1,12 @@ import { Theme } from '@mui/material'; import Box from '@mui/material/Box'; -import CircularProgress from '@mui/material/CircularProgress/CircularProgress'; import Container from '@mui/material/Container'; import Paper from '@mui/material/Paper'; import { makeStyles } from '@mui/styles'; -import { Buffer } from 'buffer'; -import { IMarkerLayer } from 'components/map/components/MarkerClusterControls'; -import { IStaticLayer } from 'components/map/components/StaticLayersControls'; -import MapContainer from 'components/map/MapContainer'; import { ActionToolbar } from 'components/toolbar/ActionToolbars'; -import { ALL_OF_BC_BOUNDARY, MAP_DEFAULT_ZOOM, SPATIAL_COMPONENT_TYPE } from 'constants/spatial'; -import { DialogContext } from 'contexts/dialogContext'; -import { Feature } from 'geojson'; import { useApi } from 'hooks/useApi'; import useDataLoader from 'hooks/useDataLoader'; -import useDataLoaderError from 'hooks/useDataLoaderError'; -import { LatLngBounds, LatLngBoundsExpression, LatLngTuple } from 'leaflet'; -import { useContext, useEffect, useState } from 'react'; -import { useHistory, useParams } from 'react-router'; -import { calculateUpdatedMapBounds } from 'utils/mapUtils'; -import { parseSpatialDataByType } from 'utils/spatial-utils'; -import DatasetArtifacts from './components/DatasetArtifacts'; -import RelatedDatasets from './components/RelatedDatasets'; -import RenderWithHandlebars from './components/RenderWithHandlebars'; +import { useParams } from 'react-router'; const useStyles = makeStyles((theme: Theme) => ({ datasetTitleContainer: { @@ -45,128 +29,130 @@ const DatasetPage: React.FC = () => { const classes = useStyles(); const biohubApi = useApi(); const urlParams = useParams(); - const dialogContext = useContext(DialogContext); - const history = useHistory(); + // const dialogContext = useContext(DialogContext); + // const history = useHistory(); const datasetId = urlParams['id']; - const datasetDataLoader = useDataLoader(() => biohubApi.dataset.getDatasetEML(datasetId)); - const templateDataLoader = useDataLoader(() => biohubApi.dataset.getHandleBarsTemplateByDatasetId(datasetId)); - - const fileDataLoader = useDataLoader((searchBoundary: Feature, searchType: string[], searchZoom: number) => - biohubApi.search.getSpatialDataFile({ - boundary: [searchBoundary], - type: searchType, - zoom: searchZoom, - datasetID: datasetId - }) - ); - - useDataLoaderError(datasetDataLoader, () => { - return { - dialogTitle: 'Error Loading Dataset', - dialogText: - 'An error has occurred while attempting to load the dataset, please try again. If the error persists, please contact your system administrator.', - onOk: () => { - datasetDataLoader.clear(); - dialogContext.setErrorDialog({ open: false }); - history.replace('/search'); - }, - onClose: () => { - datasetDataLoader.clear(); - dialogContext.setErrorDialog({ open: false }); - history.replace('/search'); - } - }; - }); + const datasetDataLoader = useDataLoader(() => biohubApi.dataset.getDataset(datasetId)); + // const templateDataLoader = useDataLoader(() => biohubApi.dataset.getHandleBarsTemplateByDatasetId(datasetId)); + + // const fileDataLoader = useDataLoader((searchBoundary: Feature, searchType: string[], searchZoom: number) => + // biohubApi.search.getSpatialDataFile({ + // boundary: [searchBoundary], + // type: searchType, + // zoom: searchZoom, + // datasetID: datasetId + // }) + // ); + + // useDataLoaderError(datasetDataLoader, () => { + // return { + // dialogTitle: 'Error Loading Dataset', + // dialogText: + // 'An error has occurred while attempting to load the dataset, please try again. If the error persists, please contact your system administrator.', + // onOk: () => { + // datasetDataLoader.clear(); + // dialogContext.setErrorDialog({ open: false }); + // history.replace('/search'); + // }, + // onClose: () => { + // datasetDataLoader.clear(); + // dialogContext.setErrorDialog({ open: false }); + // history.replace('/search'); + // } + // }; + // }); datasetDataLoader.load(); - templateDataLoader.load(); - - const mapDataLoader = useDataLoader((searchBoundary: Feature, searchType: string[], searchZoom: number) => - biohubApi.search.getSpatialData({ - boundary: [searchBoundary], - type: searchType, - zoom: searchZoom, - datasetID: datasetId - }) - ); - - useDataLoaderError(fileDataLoader, () => { - return { - dialogTitle: 'Error Exporting Data', - dialogText: - 'An error has occurred while attempting to archive and download occurrence data, please try again. If the error persists, please contact your system administrator.' - }; - }); - - useDataLoaderError(mapDataLoader, () => { - return { - dialogTitle: 'Error Loading Map Data', - dialogText: - 'An error has occurred while attempting to load the map data, please try again. If the error persists, please contact your system administrator.' - }; - }); - - const [markerLayers, setMarkerLayers] = useState([]); - const [staticLayers, setStaticLayers] = useState([]); - const [mapBoundary, setMapBoundary] = useState(undefined); - - useEffect(() => { - if (!fileDataLoader.data) { - return; - } - - const data = fileDataLoader.data; - - const content = Buffer.from(data, 'hex'); - - const blob = new Blob([content], { type: 'application/zip' }); - - const link = document.createElement('a'); - - link.download = `${datasetId}.zip`; - - link.href = URL.createObjectURL(blob); - - link.click(); - - URL.revokeObjectURL(link.href); - }, [datasetId, fileDataLoader.data]); - - useEffect(() => { - if (!mapDataLoader.data) { - return; - } - - const result = parseSpatialDataByType(mapDataLoader.data); - if (result.staticLayers[0]?.features[0]?.geoJSON) { - const bounds = calculateUpdatedMapBounds([result.staticLayers[0].features[0].geoJSON]); - if (bounds) { - const newBounds = new LatLngBounds(bounds[0] as LatLngTuple, bounds[1] as LatLngTuple); - setMapBoundary(newBounds); - } - } - setStaticLayers(result.staticLayers); - setMarkerLayers(result.markerLayers); - }, [mapDataLoader.data]); - - mapDataLoader.load( - ALL_OF_BC_BOUNDARY, - [SPATIAL_COMPONENT_TYPE.BOUNDARY, SPATIAL_COMPONENT_TYPE.OCCURRENCE], - MAP_DEFAULT_ZOOM - ); - - if (!datasetDataLoader.data || !templateDataLoader.data) { - return ; - } + console.log('datasetDataLoader.data', datasetDataLoader.data); + // // templateDataLoader.load(); + + // const mapDataLoader = useDataLoader((searchBoundary: Feature, searchType: string[], searchZoom: number) => + // biohubApi.search.getSpatialData({ + // boundary: [searchBoundary], + // type: searchType, + // zoom: searchZoom, + // datasetID: datasetId + // }) + // ); + + // useDataLoaderError(fileDataLoader, () => { + // return { + // dialogTitle: 'Error Exporting Data', + // dialogText: + // 'An error has occurred while attempting to archive and download occurrence data, please try again. If the error persists, please contact your system administrator.' + // }; + // }); + + // useDataLoaderError(mapDataLoader, () => { + // return { + // dialogTitle: 'Error Loading Map Data', + // dialogText: + // 'An error has occurred while attempting to load the map data, please try again. If the error persists, please contact your system administrator.' + // }; + // }); + + // const [markerLayers, setMarkerLayers] = useState([]); + // const [staticLayers, setStaticLayers] = useState([]); + // const [mapBoundary, setMapBoundary] = useState(undefined); + + // useEffect(() => { + // if (!fileDataLoader.data) { + // return; + // } + + // const data = fileDataLoader.data; + + // const content = Buffer.from(data, 'hex'); + + // const blob = new Blob([content], { type: 'application/zip' }); + + // const link = document.createElement('a'); + + // link.download = `${datasetId}.zip`; + + // link.href = URL.createObjectURL(blob); + + // link.click(); + + // URL.revokeObjectURL(link.href); + // }, [datasetId, fileDataLoader.data]); + + // useEffect(() => { + // if (!mapDataLoader.data) { + // return; + // } + + // const result = parseSpatialDataByType(mapDataLoader.data); + // if (result.staticLayers[0]?.features[0]?.geoJSON) { + // const bounds = calculateUpdatedMapBounds([result.staticLayers[0].features[0].geoJSON]); + // if (bounds) { + // const newBounds = new LatLngBounds(bounds[0] as LatLngTuple, bounds[1] as LatLngTuple); + // setMapBoundary(newBounds); + // } + // } + // setStaticLayers(result.staticLayers); + // setMarkerLayers(result.markerLayers); + // }, [mapDataLoader.data]); + + // mapDataLoader.load( + // ALL_OF_BC_BOUNDARY, + // [SPATIAL_COMPONENT_TYPE.BOUNDARY, SPATIAL_COMPONENT_TYPE.OCCURRENCE], + // MAP_DEFAULT_ZOOM + // ); + + // if (!datasetDataLoader.data) { + // // || !templateDataLoader.data) { + // return ; + // } return ( - + {/* */} @@ -180,10 +166,10 @@ const DatasetPage: React.FC = () => { /> - + {/* */} - = () => { doubleClickZoomEnabled={false} draggingEnabled={true} layerControlEnabled={false} - /> + /> */} - - - + {/* */} - - - + {/* */} diff --git a/app/src/features/datasets/DatasetsRouter.tsx b/app/src/features/datasets/DatasetsRouter.tsx index 016330fc3..a7c016027 100644 --- a/app/src/features/datasets/DatasetsRouter.tsx +++ b/app/src/features/datasets/DatasetsRouter.tsx @@ -13,7 +13,11 @@ const DatasetsRouter: React.FC = () => { - + {/* + + */} + + diff --git a/app/src/features/search/SearchComponent.tsx b/app/src/features/search/SearchComponent.tsx index d606a65b5..d9f8c1ab7 100644 --- a/app/src/features/search/SearchComponent.tsx +++ b/app/src/features/search/SearchComponent.tsx @@ -8,8 +8,9 @@ import InputAdornment from '@mui/material/InputAdornment'; import { makeStyles } from '@mui/styles'; import { useFormikContext } from 'formik'; import { IAdvancedSearch } from 'interfaces/useSearchApi.interface'; +import { ChangeEvent } from 'react'; -const useStyles = makeStyles((theme: Theme) => ({ +export const useSearchInputStyles = makeStyles((theme: Theme) => ({ searchInputContainer: { position: 'relative' }, @@ -59,8 +60,35 @@ const useStyles = makeStyles((theme: Theme) => ({ } })); +interface ISearchInputProps { + placeholderText: string; + handleChange: (e: ChangeEvent) => void; + value: string; +} + +export const SearchInput = (props: ISearchInputProps) => { + const classes = useSearchInputStyles(); + return ( + + + + } + disableUnderline={true} + placeholder={props.placeholderText} + onChange={props.handleChange} + value={props.value} + /> + ); +}; + const SearchComponent: React.FC = () => { - const classes = useStyles(); + const classes = useSearchInputStyles(); const formikProps = useFormikContext(); const { handleSubmit, handleChange, values } = formikProps; @@ -68,19 +96,9 @@ const SearchComponent: React.FC = () => { return (
- - - - } - disableUnderline={true} - placeholder="Enter a species name or keyword" - onChange={handleChange} + + )} + {(dataset.item.security === SECURITY_APPLIED_STATUS.UNSECURED || + dataset.item.security === SECURITY_APPLIED_STATUS.PARTIALLY_SECURED) && ( + + )} + + } + /> + + + {highlight( + dataset.item.description, + dataset?.matches?.find((match) => match.key === 'description')?.indices + )} + + + + ))} + + + + + ); +}; + +export default SubmissionsListPage; diff --git a/app/src/features/submissions/SubmissionsRouter.tsx b/app/src/features/submissions/SubmissionsRouter.tsx new file mode 100644 index 000000000..6a53eb540 --- /dev/null +++ b/app/src/features/submissions/SubmissionsRouter.tsx @@ -0,0 +1,28 @@ +import { Redirect, Route, Switch } from 'react-router'; +import RouteWithTitle from 'utils/RouteWithTitle'; +import { getTitle } from 'utils/Utils'; +import DatasetListPage from './SubmissionsListPage'; + +/** + * Router for all `/submissions/*` pages. + * + * @return {*} + */ +const SubmissionsRouter = () => { + return ( + + {/* */} + + + + + + {/* Catch any unknown routes, and re-direct to the not found page */} + + + + + ); +}; + +export default SubmissionsRouter; diff --git a/app/src/features/submissions/components/SubmissionsListSortMenu.tsx b/app/src/features/submissions/components/SubmissionsListSortMenu.tsx new file mode 100644 index 000000000..124d5db6f --- /dev/null +++ b/app/src/features/submissions/components/SubmissionsListSortMenu.tsx @@ -0,0 +1,96 @@ +import { mdiChevronDown, mdiSortAlphabeticalAscending, mdiSortAlphabeticalDescending } from '@mdi/js'; +import Icon from '@mdi/react'; +import Button from '@mui/material/Button'; +import Menu from '@mui/material/Menu'; +import MenuItem from '@mui/material/MenuItem'; +import useTheme from '@mui/system/useTheme'; +import { FuseResult } from 'fuse.js'; +import { ISubmission } from 'interfaces/useSubmissionsApi.interface'; +import sortBy from 'lodash-es/sortBy'; +import { useState } from 'react'; + +type SortBy = 'asc' | 'desc'; + +interface ISubmissionsListSortMenuProps { + data: FuseResult[]; + handleSortedFuzzyData: (data: FuseResult[]) => void; +} + +/** + * Renders 'Sort By' button for SubmissionsListPage + * Note: currently supports title and date sorting + * + * @param {ISubmissionsListSortMenuProps} props + * @returns {*} + */ +const SubmissionsListSortMenu = (props: ISubmissionsListSortMenuProps) => { + const theme = useTheme(); + + const [anchorEl, setAnchorEl] = useState(null); + + const open = Boolean(anchorEl); + + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + + const handleClose = () => { + setAnchorEl(null); + }; + + /** + * sorts datasets by property + * + * @param {keyof IDataset} sortKey - property to sort datasets by + * @param {SortBy} [sortDirection] - ascending or descending sort + */ + const handleSort = (sortKey: keyof ISubmission, sortDirection: SortBy) => { + const sortedData = + sortDirection === 'asc' + ? sortBy(props.data, (fuzzyDataset) => fuzzyDataset.item[sortKey]) + : sortBy(props.data, (fuzzyDataset) => fuzzyDataset.item[sortKey]).reverse(); + props.handleSortedFuzzyData(sortedData); + handleClose(); + }; + + const sortMenuItem = (sortKey: keyof ISubmission, itemName: string, sortBy: SortBy = 'asc') => { + const label = `${itemName} ${sortBy === 'asc' ? 'ascending' : 'descending'}`; + return ( + { + handleSort(sortKey, sortBy); + }} + dense> + + {label} + + ); + }; + + return ( +
+ + + {sortMenuItem('name', 'Title')} + {sortMenuItem('name', 'Title', 'desc')} + {sortMenuItem('submission_date', 'Date')} + {sortMenuItem('submission_date', 'Date', 'desc')} + +
+ ); +}; + +export default SubmissionsListSortMenu; diff --git a/app/src/hooks/api/useDatasetApi.ts b/app/src/hooks/api/useDatasetApi.ts index 7f7a4ebb5..af02498fb 100644 --- a/app/src/hooks/api/useDatasetApi.ts +++ b/app/src/hooks/api/useDatasetApi.ts @@ -1,9 +1,9 @@ import { AxiosInstance } from 'axios'; import { IArtifact, - IDatasetForReview, IHandlebarsTemplates, - IListRelatedDatasetsResponse + IListRelatedDatasetsResponse, + IUnreviewedSubmission } from 'interfaces/useDatasetApi.interface'; import { IKeywordSearchResponse } from 'interfaces/useSearchApi.interface'; @@ -26,12 +26,12 @@ const useDatasetApi = (axios: AxiosInstance) => { }; /** - * Fetch all unsecure datasets for review. + * Fetch all submissions that have not completed security review. * - * @returns {*} {Promise} + * @return {*} {Promise} */ - const listAllDatasetsForReview = async (): Promise => { - const { data } = await axios.get(`api/administrative/review/list`); + const getUnreviewedSubmissions = async (): Promise => { + const { data } = await axios.get(`api/administrative/submission/unreviewed`); return data; }; @@ -48,6 +48,19 @@ const useDatasetApi = (axios: AxiosInstance) => { return data; }; + /** + * Fetch dataset data by datasetUUID. + * + * @param {string} datasetUUID + * @return {*} {Promise} + */ + const getDataset = async (datasetUUID: string): Promise => { + const { data } = await axios.get(`api/dataset/${datasetUUID}`); + console.log('data', data); + + return data; + }; + /** * Fetch dataset artifacts by datasetId. * @@ -97,8 +110,9 @@ const useDatasetApi = (axios: AxiosInstance) => { return { listAllDatasets, - listAllDatasetsForReview, + getUnreviewedSubmissions, getDatasetEML, + getDataset, getDatasetArtifacts, getArtifactSignedUrl, getHandleBarsTemplateByDatasetId, diff --git a/app/src/hooks/api/useSubmissionsApi.ts b/app/src/hooks/api/useSubmissionsApi.ts index bf9ffad19..a6cf1e855 100644 --- a/app/src/hooks/api/useSubmissionsApi.ts +++ b/app/src/hooks/api/useSubmissionsApi.ts @@ -1,5 +1,6 @@ import { AxiosInstance } from 'axios'; -import { IListSubmissionsResponse } from 'interfaces/useSubmissionsApi.interface'; +import { SECURITY_APPLIED_STATUS } from 'interfaces/useDatasetApi.interface'; +import { IListSubmissionsResponse, ISubmission } from 'interfaces/useSubmissionsApi.interface'; /** * Returns a set of supported CRUD api methods submissions. @@ -30,16 +31,59 @@ const useSubmissionsApi = (axios: AxiosInstance) => { return data; }; + // Triggers a request to index the given submission const test = async (submissionId: number): Promise => { const { data } = await axios.post(`/api/dataset/search-idx?submissionId=${submissionId}`); return data; } + /** NET-NEW FRONTEND REQUESTS FOR UPDATED SCHEMA **/ + + /** + * Fetch list of all reviewed submissions + * NOTE: mock implementation + * TODO: return real data once api endpoint created + * + * @async + * @returns {*} {Promise} + */ + const listReviewedSubmissions = async (): Promise => { + const keywords = ['moose', 'caribou', 'deer', 'bear', 'bat']; + const securityLevel = { + 0: SECURITY_APPLIED_STATUS.SECURED, + 1: SECURITY_APPLIED_STATUS.UNSECURED, + 2: SECURITY_APPLIED_STATUS.SECURED, + 3: SECURITY_APPLIED_STATUS.PARTIALLY_SECURED, + 4: SECURITY_APPLIED_STATUS.PARTIALLY_SECURED + }; + return keywords.map((keyword, idx) => ({ + submission_id: idx + 1, + submission_feature_id: idx, + name: `Dataset - ${keyword}`, + description: `${keywords[idx] + 1 ?? 'test'} Lorem ipsum dolor sit amet, consectetur adipiscing elit. ${keyword}`, + submission_date: new Date(Date.now() - 86400000 * (300 * idx)), + security: securityLevel[idx] + })); + }; + + /** + * repackages and retrieves json data from self and each child under submission + * Note: unknown how this will work with artifacts. SignedURL? + * + * @async + * @returns {Promise} json data repackaged from each level of children + */ + const getSubmissionDownloadPackage = async (): Promise => { + return { mockJson: 'mockValue' }; + }; + return { listSubmissions, getSignedUrl, - test + test, + listReviewedSubmissions, + getSubmissionDownloadPackage }; }; diff --git a/app/src/hooks/useDebounce.test.tsx b/app/src/hooks/useDebounce.test.tsx new file mode 100644 index 000000000..67975049b --- /dev/null +++ b/app/src/hooks/useDebounce.test.tsx @@ -0,0 +1,21 @@ +import { act, renderHook } from '@testing-library/react-hooks'; +import { waitFor } from 'test-helpers/test-utils'; +import useDebounce from './useDebounce'; + +const mockCallback = jest.fn(); + +describe('useDebounce', () => { + it('should debounce repeated calls', async () => { + const { result } = renderHook(() => useDebounce(mockCallback, 500)); + const debounce = result.current; + act(() => debounce()); + await waitFor(() => { + expect(mockCallback.mock.calls[0]).toBeDefined(); + }); + // this request should fail as it is being requested too quickly + act(() => debounce()); + await waitFor(() => { + expect(mockCallback.mock.calls[1]).not.toBeDefined(); + }); + }); +}); diff --git a/app/src/hooks/useDebounce.tsx b/app/src/hooks/useDebounce.tsx new file mode 100644 index 000000000..b8cee4265 --- /dev/null +++ b/app/src/hooks/useDebounce.tsx @@ -0,0 +1,29 @@ +import debounce from 'lodash-es/debounce'; +import { useEffect, useMemo, useRef } from 'react'; + +/** + * hook to delay multiple repeditive executions of callback + * + * @param {() => void} callback - function to fire + * @param {number} [msDelay] - milliseconds to delay callback fire + * @returns {DebouncedFunc<() => void>} - debounced callback + */ +const useDebounce = (callback: () => void, msDelay = 500) => { + const ref = useRef(callback); + + useEffect(() => { + ref.current = callback; + }, [callback]); + + const debouncedCallback = useMemo(() => { + const func = () => { + ref.current?.(); + }; + + return debounce(func, msDelay); + }, [msDelay]); + + return debouncedCallback; +}; + +export default useDebounce; diff --git a/app/src/hooks/useDownloadJSON.tsx b/app/src/hooks/useDownloadJSON.tsx new file mode 100644 index 000000000..69314ff69 --- /dev/null +++ b/app/src/hooks/useDownloadJSON.tsx @@ -0,0 +1,25 @@ +const useDownloadJSON = () => { + /** + * hook to handle downloading raw data as JSON + * Note: currently this does not zip the file. Can be modified if needed. + * + * @param {any} data - to download + * @param {string} fileName - name of file excluding file extension ie: file1 + */ + const download = (data: any, fileName: string) => { + const blob = new Blob([JSON.stringify(data, undefined, 2)], { type: 'application/json' }); + + const link = document.createElement('a'); + + link.download = `${fileName}.json`; + + link.href = URL.createObjectURL(blob); + + link.click(); + + URL.revokeObjectURL(link.href); + }; + return download; +}; + +export default useDownloadJSON; diff --git a/app/src/hooks/useFuzzySearch.test.tsx b/app/src/hooks/useFuzzySearch.test.tsx new file mode 100644 index 000000000..4afdf6287 --- /dev/null +++ b/app/src/hooks/useFuzzySearch.test.tsx @@ -0,0 +1,102 @@ +import { act, renderHook } from '@testing-library/react-hooks'; +import { FuseResult } from 'fuse.js'; +import useFuzzySearch from './useFuzzySearch'; + +const item = { a: 'zzz', b: 'hello world' }; +const mockDataArray = [item]; + +const mockFuzzyData: FuseResult[] = [ + { + item, + matches: [], + refIndex: 0 + } +]; + +const mockOptions = { minMatchCharLength: 5 }; + +describe('useFuzzySearch', () => { + describe('mounting conditions', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, {})); + + it('should mount with empty search string', () => { + expect(result.current.searchValue).toBe(''); + }); + + it('should mount with fuzzyData array in FuseResult structure', () => { + expect(result.current.fuzzyData).toStrictEqual(mockFuzzyData); + }); + }); + describe('handleFuzzyData', () => { + it('should set fuzzyData with new array', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, {})); + act(() => result.current.handleFuzzyData([{ item: { a: 'test', b: 'test' }, matches: [], refIndex: 0 }])); + expect(result.current.fuzzyData[0].item.a).toBe('test'); + }); + }); + + describe('handleSearch', () => { + it('should set searchValue', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, {})); + act(() => result.current.handleSearch({ target: { value: 'test' } } as any)); + expect(result.current.searchValue).toBe('test'); + }); + + it('should setFuzzyData to default when no search value provided', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, {})); + act(() => result.current.handleSearch({ target: { value: '' } } as any)); + expect(result.current.searchValue).toBe(''); + expect(result.current.fuzzyData).toStrictEqual(mockFuzzyData); + }); + + it('should setFuzzyData to default when no search value provided', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, {})); + act(() => result.current.handleSearch({ target: { value: '' } } as any)); + expect(result.current.searchValue).toBe(''); + expect(result.current.fuzzyData).toStrictEqual(mockFuzzyData); + }); + + it('should setFuzzyData to default when character count is less than minMatchCharLength', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, mockOptions)); + act(() => result.current.handleSearch({ target: { value: 'aaaa' } } as any)); + expect(result.current.searchValue).toBe('aaaa'); + expect(result.current.fuzzyData).toStrictEqual(mockFuzzyData); + }); + }); + describe('highlight', () => { + const { result } = renderHook(() => useFuzzySearch(mockDataArray, { highlightColour: '#ffff' })); + it('should return formatted html if highlight indices provided', () => { + act(() => { + const jsx = result.current.highlight('abc', [[0, 1]]); + const shouldEqual = ( + <> + abc + + ); + expect(jsx.toString()).toEqual(shouldEqual.toString()); + }); + }); + + it('should return string value if no indices', () => { + act(() => { + const jsx = result.current.highlight('abc', []); + expect(jsx.toString()).toEqual('abc'); + }); + }); + + it('should highlight whole string', () => { + act(() => { + const jsx = result.current.highlight('abc', [ + [0, 1], + [2, 2] + ]); + const shouldEqual = ( + <> + abc + + ); + expect(jsx.toString()).toEqual(shouldEqual.toString()); + }); + }); + }); +}); diff --git a/app/src/hooks/useFuzzySearch.tsx b/app/src/hooks/useFuzzySearch.tsx new file mode 100644 index 000000000..fbab89ee9 --- /dev/null +++ b/app/src/hooks/useFuzzySearch.tsx @@ -0,0 +1,144 @@ +import Fuse, { FuseResult, IFuseOptions, RangeTuple } from 'fuse.js'; +import { ChangeEvent, useEffect, useState } from 'react'; +import useDebounce from './useDebounce'; + +interface IUseFuzzyOptions extends IFuseOptions { + highlightColour?: string; + debounceDelayMs?: number; +} + +/** + * hook to consolidate common fuzzy finding utilities + * 1. fuzzy finding against data set + * 2. handling search value + * 3. filtering data set + * 4. highlighting with matched indices + * + * @template T - object of some type + * @param {T[] | undefined} data - dataset to fuzzy find against + * @param {IUseFuzzyOptions} options - fuse options + additional customizations + * @returns {*} + */ +const useFuzzySearch = (data: T[] | undefined, options: IUseFuzzyOptions) => { + const { highlightColour, debounceDelayMs, ...customOptions } = options; + const defaultFuzzyOptions: IFuseOptions = { + // keys to search object array for + // prioritizes shorter key names by default ie: title > description + keys: [], + // starting location of search ie: index 100 + location: 100, + // distance from location value can be + distance: 200, + // calculates exact match with location and distance + threshold: 0.5, + // only run the fuzzy search when more than 2 characters entered + minMatchCharLength: 3, + // provides the match indices used for highlighting + includeMatches: true, + // extends the search to use logical query operators + useExtendedSearch: true + }; + + const optionsOverride = { ...defaultFuzzyOptions, ...customOptions }; + + const [searchValue, setSearchValue] = useState(''); + const [defaultFuzzyData, setDefaultFuzzyData] = useState[]>([]); + + const [fuzzyData, setFuzzyData] = useState[]>([]); + + /** + * set fuzzyData and defaultFuzzyData + * onMount / change of data generate fuzzyData of same structure fuse expects + * useful for having a single array to render + * + */ + useEffect(() => { + const dataAsFuzzy = + data?.map((item, refIndex) => ({ + item, + refIndex, + matches: [] + })) ?? []; + setDefaultFuzzyData(dataAsFuzzy); + setFuzzyData(dataAsFuzzy); + }, [data]); + + /** + * handles fuzzy finding a value for data and sorting new fuzzy data + * + * @param {string} value - string to fuzzy find against + */ + const handleFuzzy = (value: string) => { + if (!value || (optionsOverride?.minMatchCharLength && value.length < optionsOverride?.minMatchCharLength)) { + setFuzzyData(defaultFuzzyData); + return; + } + + const fuse = new Fuse(data ?? [], optionsOverride); + + /** + * modify the value to include the fuse.js logical 'OR' operator + * ie: 'moose | dataset' + * + */ + const searchValue = value.replaceAll(' ', ' | '); + const fuzzyDatasets = fuse.search(searchValue); + setFuzzyData(fuzzyDatasets); + }; + + /** + * delay repeatedly calling handleFuzzy + * + */ + const debounceFuzzySearch = useDebounce(() => { + handleFuzzy(searchValue); + }, debounceDelayMs ?? 350); + + /** + * sets the search value and debounces the immediate following requests + * ie: as user types, it only fuzzy finds/sorts after a set amount of ms + * + * @param {ChangeEvent} event + */ + const handleSearch = (event: ChangeEvent) => { + const value: string = event.target.value; + + setSearchValue(value); + + debounceFuzzySearch(); + }; + + /** + * highlights sections of a string from array of start/end indices + * ex: <>hello world!<> + * @param {string} value - string to highlight + * @param {readonly RangeTuple[]} [indices] - array of start end indexes ie: [[0,1], [4,5]] + * @param {number} [i] - index used for recursion + * @returns {string | JSX.Element} returns string or highlighted fragment + */ + const highlight = (value: string, indices: readonly RangeTuple[] = [], i = 1): string | JSX.Element => { + const pair = indices[indices.length - i]; + return !pair ? ( + value + ) : ( + <> + {highlight(value.substring(0, pair[0]), indices, i + 1)} + {value.substring(pair[0], pair[1] + 1)} + {value.substring(pair[1] + 1)} + + ); + }; + + /** + * handler for manually setting fuzzyData, usefull if using external sorting + * + * @param {FuseResult[]} newFuzzyData + */ + const handleFuzzyData = (newFuzzyData: FuseResult[]) => { + setFuzzyData(newFuzzyData); + }; + + return { fuzzyData, handleFuzzyData, handleSearch, searchValue, highlight }; +}; + +export default useFuzzySearch; diff --git a/app/src/interfaces/useDatasetApi.interface.ts b/app/src/interfaces/useDatasetApi.interface.ts index 6a1e45612..945100fdd 100644 --- a/app/src/interfaces/useDatasetApi.interface.ts +++ b/app/src/interfaces/useDatasetApi.interface.ts @@ -24,6 +24,7 @@ export interface IArtifact { export enum SECURITY_APPLIED_STATUS { SECURED = 'SECURED', UNSECURED = 'UNSECURED', + PARTIALLY_SECURED = 'PARTIALLY_SECURED', PENDING = 'PENDING' } @@ -57,3 +58,11 @@ export interface IDatasetForReview { last_updated: string; keywords: string[]; } + +export interface IUnreviewedSubmission { + submission_id: number; + uuid: string; + name: string; + description: string; + create_date: string; +} diff --git a/app/src/interfaces/useSubmissionsApi.interface.ts b/app/src/interfaces/useSubmissionsApi.interface.ts index a4b219de4..cd1e7305f 100644 --- a/app/src/interfaces/useSubmissionsApi.interface.ts +++ b/app/src/interfaces/useSubmissionsApi.interface.ts @@ -1,3 +1,5 @@ +import { SECURITY_APPLIED_STATUS } from './useDatasetApi.interface'; + export type IListSubmissionsResponse = Array<{ submission_id: number; submission_status: string; @@ -15,3 +17,14 @@ export type IListSubmissionsResponse = Array<{ update_user: number | null; revision_count: number; }>; + +/** NET-NEW INTERFACES FOR UPDATED SCHEMA **/ + +export interface ISubmission { + submission_id: number; + submission_feature_id: number; + name: string; + description: string; + submission_date: Date; + security: SECURITY_APPLIED_STATUS; +} diff --git a/database/package-lock.json b/database/package-lock.json index b6fd6896a..aa9d5dea2 100644 --- a/database/package-lock.json +++ b/database/package-lock.json @@ -528,24 +528,6 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "dev": true }, - "core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true - }, - "core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true - }, - "core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "dev": true - }, "create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -1503,24 +1485,6 @@ "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", "dev": true }, - "jsonparse": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", - "dev": true - }, - "jsonparse": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", - "dev": true - }, - "jsonparse": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", - "dev": true - }, "knex": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/knex/-/knex-2.4.2.tgz", @@ -2378,24 +2342,6 @@ "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "dev": true }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "dev": true - }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "dev": true - }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "dev": true - }, "tildify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", diff --git a/database/src/migrations/20231117000001_security_tables.ts b/database/src/migrations/20231117000001_security_tables.ts index d538c965c..5ea3ae00b 100644 --- a/database/src/migrations/20231117000001_security_tables.ts +++ b/database/src/migrations/20231117000001_security_tables.ts @@ -75,37 +75,8 @@ export async function up(knex: Knex): Promise { ---------------------------------------------------------------------------------------- - CREATE TABLE artifact_security( - artifact_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - artifact_id integer NOT NULL, - security_rule_id integer NOT NULL, - record_effective_date date NOT NULL, - record_end_date date, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT artifact_security_pk PRIMARY KEY (artifact_security_id) - ); - - COMMENT ON COLUMN artifact_security.artifact_security_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN artifact_security.artifact_id IS 'Foreign key to the artifact table.'; - COMMENT ON COLUMN artifact_security.security_rule_id IS 'Foreign key to the security_rule table.'; - COMMENT ON COLUMN artifact_security.record_effective_date IS 'Record level effective date.'; - COMMENT ON COLUMN artifact_security.record_end_date IS 'Record level end date.'; - COMMENT ON COLUMN artifact_security.create_date IS 'The datetime the record was created.'; - COMMENT ON COLUMN artifact_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; - COMMENT ON COLUMN artifact_security.update_date IS 'The datetime the record was updated.'; - COMMENT ON COLUMN artifact_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; - COMMENT ON COLUMN artifact_security.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE artifact_security IS 'A join table between artifact and security_rule. Defines which security rules are applied to the an artifact.'; - - ---------------------------------------------------------------------------------------- - CREATE TABLE security_string( security_string_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -122,7 +93,6 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_string.security_string_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN security_string.security_rule_id IS 'Foreign key to the security_string table.'; COMMENT ON COLUMN security_string.name IS 'The name of the security_string record.'; COMMENT ON COLUMN security_string.description IS 'The description of the security_string record.'; COMMENT ON COLUMN security_string.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -139,7 +109,6 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_number( security_number_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -156,7 +125,6 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_number.security_number_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN security_number.security_rule_id IS 'Foreign key to the security_number table.'; COMMENT ON COLUMN security_number.name IS 'The name of the security_number record.'; COMMENT ON COLUMN security_number.description IS 'The description of the security_number record.'; COMMENT ON COLUMN security_number.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -173,7 +141,6 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_datetime( security_datetime_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -190,7 +157,6 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_datetime.security_datetime_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN security_datetime.security_rule_id IS 'Foreign key to the security_datetime table.'; COMMENT ON COLUMN security_datetime.name IS 'The name of the security_datetime record.'; COMMENT ON COLUMN security_datetime.description IS 'The description of the security_datetime record.'; COMMENT ON COLUMN security_datetime.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -207,7 +173,6 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_spatial( security_spatial_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -224,7 +189,6 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_spatial.security_spatial_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN security_spatial.security_rule_id IS 'Foreign key to the security_spatial table.'; COMMENT ON COLUMN security_spatial.name IS 'The name of the security_spatial record.'; COMMENT ON COLUMN security_spatial.description IS 'The description of the security_spatial record.'; COMMENT ON COLUMN security_spatial.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -265,27 +229,6 @@ export async function up(knex: Knex): Promise { CREATE INDEX submission_feature_security_idx2 ON submission_feature_security(security_rule_id); - ---------------------------------------------------------------------------------------- - -- Create Indexes and Constraints for table: artifact_security - ---------------------------------------------------------------------------------------- - - -- Add unique end-date key constraint (don't allow 2 records with the same artifact_id, security_rule_id, and a NULL record_end_date) - CREATE UNIQUE INDEX artifact_security_nuk1 ON artifact_security(artifact_id, security_rule_id, (record_end_date is NULL)) where record_end_date is null; - - -- Add foreign key constraint - ALTER TABLE artifact_security ADD CONSTRAINT artifact_security_fk1 - FOREIGN KEY (artifact_id) - REFERENCES artifact(artifact_id); - - ALTER TABLE artifact_security ADD CONSTRAINT artifact_security_fk2 - FOREIGN KEY (security_rule_id) - REFERENCES security_rule(security_rule_id); - - -- add indexes for foreign keys - CREATE INDEX artifact_security_idx1 ON artifact_security(artifact_id); - - CREATE INDEX artifact_security_idx2 ON artifact_security(security_rule_id); - ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_string ---------------------------------------------------------------------------------------- @@ -293,14 +236,6 @@ export async function up(knex: Knex): Promise { -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) CREATE UNIQUE INDEX security_string_nuk1 ON security_string(name, (record_end_date is NULL)) where record_end_date is null; - -- Add foreign key constraint - ALTER TABLE security_string ADD CONSTRAINT security_string_fk1 - FOREIGN KEY (security_rule_id) - REFERENCES security_rule(security_rule_id); - - -- add indexes for foreign keys - CREATE INDEX security_string_idx1 ON security_string(security_rule_id); - ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_number ---------------------------------------------------------------------------------------- @@ -308,14 +243,6 @@ export async function up(knex: Knex): Promise { -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) CREATE UNIQUE INDEX security_number_nuk1 ON security_number(name, (record_end_date is NULL)) where record_end_date is null; - -- Add foreign key constraint - ALTER TABLE security_number ADD CONSTRAINT security_number_fk1 - FOREIGN KEY (security_rule_id) - REFERENCES security_rule(security_rule_id); - - -- add indexes for foreign keys - CREATE INDEX security_number_idx1 ON security_number(security_rule_id); - ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_datetime ---------------------------------------------------------------------------------------- @@ -323,14 +250,6 @@ export async function up(knex: Knex): Promise { -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) CREATE UNIQUE INDEX security_datetime_nuk1 ON security_datetime(name, (record_end_date is NULL)) where record_end_date is null; - -- Add foreign key constraint - ALTER TABLE security_datetime ADD CONSTRAINT security_datetime_fk1 - FOREIGN KEY (security_rule_id) - REFERENCES security_rule(security_rule_id); - - -- add indexes for foreign keys - CREATE INDEX security_datetime_idx1 ON security_datetime(security_rule_id); - ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_spatial ---------------------------------------------------------------------------------------- @@ -338,14 +257,6 @@ export async function up(knex: Knex): Promise { -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) CREATE UNIQUE INDEX security_spatial_nuk1 ON security_spatial(name, (record_end_date is NULL)) where record_end_date is null; - -- Add foreign key constraint - ALTER TABLE security_spatial ADD CONSTRAINT security_spatial_fk1 - FOREIGN KEY (security_rule_id) - REFERENCES security_rule(security_rule_id); - - -- add indexes for foreign keys - CREATE INDEX security_spatial_idx1 ON security_spatial(security_rule_id); - ---------------------------------------------------------------------------------------- -- Create audit and journal triggers ---------------------------------------------------------------------------------------- diff --git a/database/src/migrations/release.0.8.0/biohub.sql b/database/src/migrations/release.0.8.0/biohub.sql index 0ae040fa9..c7aa6e9ce 100644 --- a/database/src/migrations/release.0.8.0/biohub.sql +++ b/database/src/migrations/release.0.8.0/biohub.sql @@ -1,43 +1,3 @@ --- --- TABLE: artifact --- - -CREATE TABLE artifact( - artifact_id integer NOT NULL, - submission_id integer NOT NULL, - uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, - file_name varchar(300) NOT NULL, - file_type varchar(300) NOT NULL, - title varchar(300), - description varchar(3000), - file_size integer, - key varchar(1000), - security_review_timestamp timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT artifact_pk PRIMARY KEY (artifact_id) -); - -COMMENT ON COLUMN artifact.artifact_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.'; -COMMENT ON COLUMN artifact.submission_id IS 'System generated surrogate primary key identifier.'; -COMMENT ON COLUMN artifact.uuid IS 'The universally unique identifier for the record.'; -COMMENT ON COLUMN artifact.file_name IS 'The name of the artifact.'; -COMMENT ON COLUMN artifact.file_type IS 'The artifact type. Artifact type examples include video, audio and field data.'; -COMMENT ON COLUMN artifact.title IS 'The title of the artifact.'; -COMMENT ON COLUMN artifact.description IS 'The description of the record.'; -COMMENT ON COLUMN artifact.file_size IS 'The size of the artifact in bytes.'; -COMMENT ON COLUMN artifact.key IS 'The identifying key to the file in the storage system.'; -COMMENT ON COLUMN artifact.security_review_timestamp IS 'The timestamp that the security review of the submission artifact was completed.'; -COMMENT ON COLUMN artifact.create_date IS 'The datetime the record was created.'; -COMMENT ON COLUMN artifact.create_user IS 'The id of the user who created the record as identified in the system user table.'; -COMMENT ON COLUMN artifact.update_date IS 'The datetime the record was updated.'; -COMMENT ON COLUMN artifact.update_user IS 'The id of the user who updated the record as identified in the system user table.'; -COMMENT ON COLUMN artifact.revision_count IS 'Revision count used for concurrency control.'; -COMMENT ON TABLE artifact IS 'A listing of historical data submission artifacts.'; - -- -- TABLE: audit_log -- @@ -67,20 +27,26 @@ COMMENT ON TABLE audit_log IS 'Holds record level audit log data for the entire -- CREATE TABLE submission( - submission_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, - publish_timestamp timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, + submission_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, + security_review_timestamp timestamptz(6), + source_system varchar(200) NOT NULL, + name varchar(200) NOT NULL, + description varchar(3000), + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT submission_pk PRIMARY KEY (submission_id) ); COMMENT ON COLUMN submission.submission_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN submission.uuid IS 'The universally unique identifier for the submission as supplied by the source system.'; -COMMENT ON COLUMN submission.publish_timestamp IS 'The timestamp of when the submission was published. Null indicates the submission is not published.'; +COMMENT ON COLUMN submission.security_review_timestamp IS 'The timestamp of when the security review of the submission was completed. Null indicates the security review has not been completed.'; +COMMENT ON COLUMN submission.source_system IS 'The name of the source system from which the submission originated.'; +COMMENT ON COLUMN submission.name IS 'The name of the submission.'; +COMMENT ON COLUMN submission.description IS 'The description of the submission.'; COMMENT ON COLUMN submission.create_date IS 'The datetime the record was created.'; COMMENT ON COLUMN submission.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN submission.update_date IS 'The datetime the record was updated.'; @@ -303,12 +269,6 @@ COMMENT ON COLUMN user_identity_source.update_user IS 'The id of the user who up COMMENT ON COLUMN user_identity_source.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE user_identity_source IS 'The source of the user identifier. This source is traditionally the system that authenticates the user. Example sources could include IDIR, BCEID and DATABASE.'; --- --- INDEX: "artifact_idx1" --- - -CREATE INDEX artifact_idx1 ON artifact(submission_id); - -- -- INDEX: submission_uk1 -- @@ -373,14 +333,6 @@ CREATE INDEX system_user_role_idx2 ON system_user_role(system_role_id); CREATE UNIQUE INDEX user_identity_source_nuk1 ON user_identity_source(name, (record_end_date is NULL)) where record_end_date is null; --- --- TABLE: artifact --- - -ALTER TABLE artifact ADD CONSTRAINT artifact_fk1 - FOREIGN KEY (submission_id) - REFERENCES submission(submission_id); - -- -- TABLE: submission_job_queue -- diff --git a/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql b/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql index ec7319f55..12855534a 100644 --- a/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql +++ b/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql @@ -1,4 +1,3 @@ -create trigger audit_artifact before insert or update or delete on biohub.artifact for each row execute procedure tr_audit_trigger(); create trigger audit_submission before insert or update or delete on biohub.submission for each row execute procedure tr_audit_trigger(); create trigger audit_system_user before insert or update or delete on biohub.system_user for each row execute procedure tr_audit_trigger(); create trigger audit_system_constant before insert or update or delete on biohub.system_constant for each row execute procedure tr_audit_trigger(); diff --git a/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql b/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql index fe7fefbee..69857ebda 100644 --- a/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql +++ b/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql @@ -1,4 +1,3 @@ -create trigger journal_artifact after insert or update or delete on biohub.artifact for each row execute procedure tr_journal_trigger(); create trigger journal_submission after insert or update or delete on biohub.submission for each row execute procedure tr_journal_trigger(); create trigger journal_system_user after insert or update or delete on biohub.system_user for each row execute procedure tr_journal_trigger(); create trigger journal_system_constant after insert or update or delete on biohub.system_constant for each row execute procedure tr_journal_trigger(); diff --git a/database/src/seeds/02_populate_feature_tables.ts b/database/src/seeds/02_populate_feature_tables.ts index 5174e9fc6..aae7d962e 100644 --- a/database/src/seeds/02_populate_feature_tables.ts +++ b/database/src/seeds/02_populate_feature_tables.ts @@ -38,9 +38,11 @@ export async function seed(knex: Knex): Promise { insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('count', 'Count', 'The count of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('latitude', 'Latitude', 'The latitude of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('longitude', 'Longitude', 'The longitude of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()) ON CONFLICT DO NOTHING; + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('s3_key', 'Key', 'The S3 storage key for an artifact', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()) ON CONFLICT DO NOTHING; -- populate feature_type table insert into feature_type (name, display_name, description, record_effective_date) values ('dataset', 'Dataset', 'A related collection of data (ie: survey)', now()) ON CONFLICT DO NOTHING; + insert into feature_type (name, display_name, description, record_effective_date) values ('artifact', 'artifact', 'An artifact (ie: image, document, pdf)', now()) ON CONFLICT DO NOTHING; insert into feature_type (name, display_name, description, record_effective_date) values ('sample_site', 'Sample Site', 'A location at which data was collected', now()) ON CONFLICT DO NOTHING; insert into feature_type (name, display_name, description, record_effective_date) values ('sample_method', 'Sample Method', 'A method used to collect data', now()) ON CONFLICT DO NOTHING; insert into feature_type (name, display_name, description, record_effective_date) values ('sample_period', 'Sample Period', 'A datetime period in which data was collected', now()) ON CONFLICT DO NOTHING; @@ -58,6 +60,9 @@ export async function seed(knex: Knex): Promise { insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'end_date'), now()) ON CONFLICT DO NOTHING; insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'geometry'), now()) ON CONFLICT DO NOTHING; + -- feature_type: artifact + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'artifact'), (select feature_property_id from feature_property where name = 's3_key'), now()) ON CONFLICT DO NOTHING; + -- feature_type: sample_site insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'name'), now()) ON CONFLICT DO NOTHING; insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'description'), now()) ON CONFLICT DO NOTHING; @@ -77,7 +82,7 @@ export async function seed(knex: Knex): Promise { -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'date_range'), now()) ON CONFLICT DO NOTHING; -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'start_date'), now()) ON CONFLICT DO NOTHING; -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'end_date'), now()) ON CONFLICT DO NOTHING; - -- insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'geometry'), now()) ON CONFLICT DO NOTHING; + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'geometry'), now()) ON CONFLICT DO NOTHING; insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'latitude'), now()) ON CONFLICT DO NOTHING; insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'longitude'), now()) ON CONFLICT DO NOTHING; insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'count'), now()) ON CONFLICT DO NOTHING; diff --git a/database/src/seeds/04_mock_test_data.ts b/database/src/seeds/04_mock_test_data.ts index 962f2c83f..8fa52980a 100644 --- a/database/src/seeds/04_mock_test_data.ts +++ b/database/src/seeds/04_mock_test_data.ts @@ -113,9 +113,9 @@ const insertDatasetRecord = async (knex: Knex, options: { submission_id: number await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); - await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); - await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); - await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + // await knex.raw(`${insertSearchStringTaxonomy({ submission_feature_id })}`); + // await knex.raw(`${insertSearchStringTaxonomy({ submission_feature_id })}`); + // await knex.raw(`${insertSearchStringTaxonomy({ submission_feature_id })}`); await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); @@ -170,10 +170,10 @@ const insertObservationRecord = async ( await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); - await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + await knex.raw(`${insertSearchStringTaxonomy({ submission_feature_id })}`); - await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); - await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); + // await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); + // await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); await knex.raw(`${insertSpatialPoint({ submission_feature_id })}`); @@ -204,7 +204,7 @@ const insertAnimalRecord = async ( await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); - await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + await knex.raw(`${insertSearchStringTaxonomy({ submission_feature_id })}`); await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); @@ -217,13 +217,17 @@ const insertAnimalRecord = async ( const insertSubmission = () => ` INSERT INTO submission ( - source_transform_id, - uuid + uuid, + name, + description, + source_system ) values ( - 1, - public.gen_random_uuid() + public.gen_random_uuid(), + $$${faker.company.name()}$$, + $$${faker.lorem.words({ min: 5, max: 100 })}$$, + 'SIMS' ) RETURNING submission_id; `; @@ -284,8 +288,8 @@ const insertSearchNumber = (options: { submission_feature_id: number }) => ` ); `; -const insertSearchTaxonomy = (options: { submission_feature_id: number }) => ` - INSERT INTO search_taxonomy +const insertSearchStringTaxonomy = (options: { submission_feature_id: number }) => ` + INSERT INTO search_string ( submission_feature_id, feature_property_id, @@ -294,7 +298,7 @@ const insertSearchTaxonomy = (options: { submission_feature_id: number }) => ` values ( ${options.submission_feature_id}, - (select feature_property_id from feature_property where name = 'number'), + (select feature_property_id from feature_property where name = 'taxonomy'), $$${faker.number.int({ min: 10000, max: 99999 })}$$ ); `; diff --git a/docker-compose.yml b/docker-compose.yml index e10f3d9f9..522be2922 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -183,6 +183,8 @@ services: - DB_SCHEMA=${DB_SCHEMA} - DB_USER_API=${DB_USER_API} - DB_USER_API_PASS=${DB_USER_API_PASS} + - ENABLE_MOCK_FEATURE_SEEDING=${ENABLE_MOCK_FEATURE_SEEDING} + - NUM_MOCK_FEATURE_SUBMISSIONS=${NUM_MOCK_FEATURE_SUBMISSIONS} volumes: - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules networks: From b5953dd937d24d1cfd98000e0538c9d82702649d Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Mon, 11 Dec 2023 14:59:35 -0800 Subject: [PATCH 08/37] SIMSBIOHUB-379: Add records to table --- api/src/services/search-index-service.ts | 30 ++++++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 251afd1f6..458dc7dab 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -25,18 +25,26 @@ export class SearchIndexService extends DBService { const stringRecords: InsertStringSearchableRecord[] = []; const featurePropertyTypeNames = await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); - - const propertyTypeMap = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { + + const featurePropertyTypeMap = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { const { property_name, ...rest } = propertyType; return [property_name, rest]; })) + + defaultLog.debug({ featurePropertyTypeMap }) features.forEach((feature) => { const { submission_feature_id } = feature; Object .entries(feature.data.properties) - .forEach(([property_name, value]) => { - const { property_type, feature_property_id } = propertyTypeMap[property_name]; + .forEach(([property_name, value]) => { + const featureProperty = featurePropertyTypeMap[property_name]; + if (!featureProperty) { + return; + } + + const { property_type, feature_property_id } = featureProperty; + switch (property_type) { case 'datetime': datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); @@ -57,6 +65,18 @@ export class SearchIndexService extends DBService { }) }); - defaultLog.debug({ label: 'indexFeaturesBySubmissionId', datetimeRecords, numberRecords, spatialRecords, stringRecords }); + + if (datetimeRecords.length) { + this.searchIndexRepository.insertSearchableDatetimeRecords(datetimeRecords); + } + if (numberRecords.length) { + this.searchIndexRepository.insertSearchableNumberRecords(numberRecords); + } + if (spatialRecords.length) { + this.searchIndexRepository.insertSearchableSpatialRecords(spatialRecords); + } + if (stringRecords.length) { + this.searchIndexRepository.insertSearchableStringRecords(stringRecords); + } } } From 2dec085b2119183cb8c6bc0722009347225c42f5 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Mon, 11 Dec 2023 15:21:11 -0800 Subject: [PATCH 09/37] SIMSBIOHUB-379: Restore admin submissions folder --- .../submission/reviewed/index.test.ts | 62 +++++++++ .../submission/reviewed/index.ts | 123 ++++++++++++++++++ .../submission/unreviewed/index.test.ts | 62 +++++++++ .../submission/unreviewed/index.ts | 123 ++++++++++++++++++ 4 files changed, 370 insertions(+) create mode 100644 api/src/paths/administrative/submission/reviewed/index.test.ts create mode 100644 api/src/paths/administrative/submission/reviewed/index.ts create mode 100644 api/src/paths/administrative/submission/unreviewed/index.test.ts create mode 100644 api/src/paths/administrative/submission/unreviewed/index.ts diff --git a/api/src/paths/administrative/submission/reviewed/index.test.ts b/api/src/paths/administrative/submission/reviewed/index.test.ts new file mode 100644 index 000000000..4119fd7e4 --- /dev/null +++ b/api/src/paths/administrative/submission/reviewed/index.test.ts @@ -0,0 +1,62 @@ +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import * as db from '../../../../database/db'; +import { HTTPError } from '../../../../errors/http-error'; +import { SubmissionService } from '../../../../services/submission-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../__mocks__/db'; +import { getReviewedSubmissionsForAdmins } from '../reviewed'; + +chai.use(sinonChai); + +describe('list', () => { + afterEach(() => { + sinon.restore(); + }); + + it('re-throws any error that is thrown', async () => { + const mockDBConnection = getMockDBConnection({ + open: () => { + throw new Error('test error'); + } + }); + + sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + const requestHandler = getReviewedSubmissionsForAdmins(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (actualError) { + expect((actualError as HTTPError).message).to.equal('test error'); + } + }); + + it('should return an array of Reviewed submission objects', async () => { + const dbConnectionObj = getMockDBConnection({ + commit: sinon.stub(), + rollback: sinon.stub(), + release: sinon.stub() + }); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + const getReviewedSubmissionsStub = sinon + .stub(SubmissionService.prototype, 'getReviewedSubmissionsForAdmins') + .resolves([]); + + const requestHandler = getReviewedSubmissionsForAdmins(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getReviewedSubmissionsStub).to.have.been.calledOnce; + expect(mockRes.statusValue).to.equal(200); + expect(mockRes.jsonValue).to.eql([]); + }); +}); diff --git a/api/src/paths/administrative/submission/reviewed/index.ts b/api/src/paths/administrative/submission/reviewed/index.ts new file mode 100644 index 000000000..83d983817 --- /dev/null +++ b/api/src/paths/administrative/submission/reviewed/index.ts @@ -0,0 +1,123 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { SYSTEM_ROLE } from '../../../../constants/roles'; +import { getDBConnection } from '../../../../database/db'; +import { defaultErrorResponses } from '../../../../openapi/schemas/http-responses'; +import { authorizeRequestHandler } from '../../../../request-handlers/security/authorization'; +import { SubmissionService } from '../../../../services/submission-service'; +import { getLogger } from '../../../../utils/logger'; + +const defaultLog = getLogger('paths/administrative/submission/reviewed'); + +export const GET: Operation = [ + authorizeRequestHandler(() => { + return { + and: [ + { + validSystemRoles: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + getReviewedSubmissionsForAdmins() +]; + +GET.apiDoc = { + description: 'Get a list of submissions that have completed security review (are reviewed).', + tags: ['admin'], + security: [ + { + Bearer: [] + } + ], + responses: { + 200: { + description: 'List of submissions that have completed security review.', + content: { + 'application/json': { + schema: { + type: 'array', + items: { + type: 'object', + properties: { + submission_id: { + type: 'integer', + minimum: 1 + }, + uuid: { + type: 'string', + format: 'uuid' + }, + security_review_timestamp: { + type: 'string', + nullable: true + }, + source_system: { + type: 'string' + }, + name: { + type: 'string', + maxLength: 200 + }, + description: { + type: 'string', + maxLength: 3000 + }, + create_date: { + type: 'string' + }, + create_user: { + type: 'integer', + minimum: 1 + }, + update_date: { + type: 'string', + nullable: true + }, + update_user: { + type: 'integer', + minimum: 1, + nullable: true + }, + revision_count: { + type: 'integer', + minimum: 0 + } + } + } + } + } + } + }, + ...defaultErrorResponses + } +}; + +/** + * Get all reviewed submissions. + * + * @returns {RequestHandler} + */ +export function getReviewedSubmissionsForAdmins(): RequestHandler { + return async (req, res) => { + const connection = getDBConnection(req['keycloak_token']); + + try { + await connection.open(); + + const service = new SubmissionService(connection); + const response = await service.getReviewedSubmissionsForAdmins(); + + await connection.commit(); + + return res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'getReviewedSubmissions', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/paths/administrative/submission/unreviewed/index.test.ts b/api/src/paths/administrative/submission/unreviewed/index.test.ts new file mode 100644 index 000000000..e5900401e --- /dev/null +++ b/api/src/paths/administrative/submission/unreviewed/index.test.ts @@ -0,0 +1,62 @@ +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { getUnreviewedSubmissionsForAdmins } from '.'; +import * as db from '../../../../database/db'; +import { HTTPError } from '../../../../errors/http-error'; +import { SubmissionService } from '../../../../services/submission-service'; +import { getMockDBConnection, getRequestHandlerMocks } from '../../../../__mocks__/db'; + +chai.use(sinonChai); + +describe('list', () => { + afterEach(() => { + sinon.restore(); + }); + + it('re-throws any error that is thrown', async () => { + const mockDBConnection = getMockDBConnection({ + open: () => { + throw new Error('test error'); + } + }); + + sinon.stub(db, 'getDBConnection').returns(mockDBConnection); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + const requestHandler = getUnreviewedSubmissionsForAdmins(); + + try { + await requestHandler(mockReq, mockRes, mockNext); + expect.fail(); + } catch (actualError) { + expect((actualError as HTTPError).message).to.equal('test error'); + } + }); + + it('should return an array of unreviewed submission objects', async () => { + const dbConnectionObj = getMockDBConnection({ + commit: sinon.stub(), + rollback: sinon.stub(), + release: sinon.stub() + }); + + sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); + + const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); + + const getUnreviewedSubmissionsStub = sinon + .stub(SubmissionService.prototype, 'getUnreviewedSubmissionsForAdmins') + .resolves([]); + + const requestHandler = getUnreviewedSubmissionsForAdmins(); + + await requestHandler(mockReq, mockRes, mockNext); + + expect(getUnreviewedSubmissionsStub).to.have.been.calledOnce; + expect(mockRes.statusValue).to.equal(200); + expect(mockRes.jsonValue).to.eql([]); + }); +}); diff --git a/api/src/paths/administrative/submission/unreviewed/index.ts b/api/src/paths/administrative/submission/unreviewed/index.ts new file mode 100644 index 000000000..5f0c2c76f --- /dev/null +++ b/api/src/paths/administrative/submission/unreviewed/index.ts @@ -0,0 +1,123 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { SYSTEM_ROLE } from '../../../../constants/roles'; +import { getDBConnection } from '../../../../database/db'; +import { defaultErrorResponses } from '../../../../openapi/schemas/http-responses'; +import { authorizeRequestHandler } from '../../../../request-handlers/security/authorization'; +import { SubmissionService } from '../../../../services/submission-service'; +import { getLogger } from '../../../../utils/logger'; + +const defaultLog = getLogger('paths/administrative/submission/unreviewed'); + +export const GET: Operation = [ + authorizeRequestHandler(() => { + return { + and: [ + { + validSystemRoles: [SYSTEM_ROLE.SYSTEM_ADMIN, SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + getUnreviewedSubmissionsForAdmins() +]; + +GET.apiDoc = { + description: 'Get a list of submissions that need security review (are unreviewed).', + tags: ['admin'], + security: [ + { + Bearer: [] + } + ], + responses: { + 200: { + description: 'List of submissions that need security review.', + content: { + 'application/json': { + schema: { + type: 'array', + items: { + type: 'object', + properties: { + submission_id: { + type: 'integer', + minimum: 1 + }, + uuid: { + type: 'string', + format: 'uuid' + }, + security_review_timestamp: { + type: 'string', + nullable: true + }, + source_system: { + type: 'string' + }, + name: { + type: 'string', + maxLength: 200 + }, + description: { + type: 'string', + maxLength: 3000 + }, + create_date: { + type: 'string' + }, + create_user: { + type: 'integer', + minimum: 1 + }, + update_date: { + type: 'string', + nullable: true + }, + update_user: { + type: 'integer', + minimum: 1, + nullable: true + }, + revision_count: { + type: 'integer', + minimum: 0 + } + } + } + } + } + } + }, + ...defaultErrorResponses + } +}; + +/** + * Get all unreviewed submissions. + * + * @returns {RequestHandler} + */ +export function getUnreviewedSubmissionsForAdmins(): RequestHandler { + return async (req, res) => { + const connection = getDBConnection(req['keycloak_token']); + + try { + await connection.open(); + + const service = new SubmissionService(connection); + const response = await service.getUnreviewedSubmissionsForAdmins(); + + await connection.commit(); + + return res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'getUnreviewedSubmissions', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} From d662267425aa60b1531e0625f4b84376dffa7f97 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Mon, 11 Dec 2023 15:48:42 -0800 Subject: [PATCH 10/37] SIMSBIOHUB-379: some type changes, debugging --- api/src/repositories/submission-repository.ts | 20 +++++++++---------- api/src/services/search-index-service.ts | 4 +++- app/src/hooks/api/useSubmissionsApi.ts | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts index a09d61941..421a3b8d2 100644 --- a/api/src/repositories/submission-repository.ts +++ b/api/src/repositories/submission-repository.ts @@ -78,7 +78,7 @@ export interface ISubmissionRecord { } export interface ISubmissionFeatureRecord { - submission_feature_id?: number; + submission_feature_id: number; submission_id: number; feature_type_id: number; data: any; // TODO: IFeatureSubmission; @@ -362,17 +362,17 @@ export class SubmissionRepository extends BaseRepository { return response.rows[0]; } - // TODO add return type - async getFeatureRecordsBySubmissionId(submissionId: number): Promise { - const queryBuilder = getKnex() - .select('*') - .from('submission_feature') - .where('submission_id', submissionId); + // TODO probably safe to remove? Duplicate of getSubmissionFeaturesBySubmissionId() + // async getFeatureRecordsBySubmissionId(submissionId: number): Promise { + // const queryBuilder = getKnex() + // .select('*') + // .from('submission_feature') + // .where('submission_id', submissionId); - const response = await this.connection.knex(queryBuilder); + // const response = await this.connection.knex(queryBuilder); - return response.rows; - } + // return response.rows; + // } /** * Get feature type id by name. diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 458dc7dab..9e2027dff 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -17,7 +17,9 @@ export class SearchIndexService extends DBService { async indexFeaturesBySubmissionId(submissionId: number): Promise { const submissionRepository = new SubmissionRepository(this.connection); - const features = await submissionRepository.getFeatureRecordsBySubmissionId(submissionId); + const features = await submissionRepository.getSubmissionFeaturesBySubmissionId(submissionId); + + defaultLog.debug({ features }) const datetimeRecords: InsertDatetimeSearchableRecord[] = []; const numberRecords: InsertNumberSearchableRecord[] = []; diff --git a/app/src/hooks/api/useSubmissionsApi.ts b/app/src/hooks/api/useSubmissionsApi.ts index 7ac7d1db9..6e86682eb 100644 --- a/app/src/hooks/api/useSubmissionsApi.ts +++ b/app/src/hooks/api/useSubmissionsApi.ts @@ -33,7 +33,7 @@ const useSubmissionsApi = (axios: AxiosInstance) => { // Triggers a request to index the given submission const test = async (submissionId: number): Promise => { - const { data } = await axios.post(`/api/dataset/search-idx?submissionId=${submissionId}`); + const { data } = await axios.post(`/api/submission/search-idx?submissionId=${submissionId}`); return data; } From c809a69f86057e49df46f28acdb5b6d3f4ff0a3f Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 12 Dec 2023 14:51:12 -0800 Subject: [PATCH 11/37] SIMSBIOHUB-379: Added timestamp to db seed --- database/src/seeds/02_populate_feature_tables.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/database/src/seeds/02_populate_feature_tables.ts b/database/src/seeds/02_populate_feature_tables.ts index aae7d962e..1b74042c2 100644 --- a/database/src/seeds/02_populate_feature_tables.ts +++ b/database/src/seeds/02_populate_feature_tables.ts @@ -34,6 +34,7 @@ export async function seed(knex: Knex): Promise { insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('date_range', 'Date Range', 'A date range', (select feature_property_type_id from feature_property_type where name = 'object'), null, now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('start_date', 'Start Date', 'The start date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('end_date', 'End Date', 'The end date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()) ON CONFLICT DO NOTHING; + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('timestamp', 'Timestamp', 'The timestamp of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), null, now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('geometry', 'Geometry', 'The location of the record', (select feature_property_type_id from feature_property_type where name = 'spatial'), null, now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('count', 'Count', 'The count of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()) ON CONFLICT DO NOTHING; insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('latitude', 'Latitude', 'The latitude of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()) ON CONFLICT DO NOTHING; From e49d60b5de49f9ff7c1593e8ab7c822f2f0ba7e7 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 12 Dec 2023 15:02:23 -0800 Subject: [PATCH 12/37] SIMSBIOHUB-379: Added some tests WIP --- .../search-index-repository.test.ts | 7 +++ api/src/services/search-index-service.test.ts | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 api/src/repositories/search-index-repository.test.ts create mode 100644 api/src/services/search-index-service.test.ts diff --git a/api/src/repositories/search-index-repository.test.ts b/api/src/repositories/search-index-repository.test.ts new file mode 100644 index 000000000..a93de9ce0 --- /dev/null +++ b/api/src/repositories/search-index-repository.test.ts @@ -0,0 +1,7 @@ + + +describe('SearchIndexRepository', () => { + describe('getFeaturePropertiesWithTypeNames', () => { + // + }) +}) diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts new file mode 100644 index 000000000..26d0e8fa7 --- /dev/null +++ b/api/src/services/search-index-service.test.ts @@ -0,0 +1,60 @@ +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { getMockDBConnection } from '../__mocks__/db'; +import { SearchIndexService } from './search-index-service'; +import { SearchIndexRepository } from '../repositories/search-index-respository'; +import { SubmissionRepository } from '../repositories/submission-repository'; + +chai.use(sinonChai); + +describe('SearchIndexService', () => { + afterEach(() => { + sinon.restore(); + }); + + describe('indexFeaturesBySubmissionId', () => { + it('should correctly index a submission', async () => { + const mockDBConnection = getMockDBConnection(); + + const searchIndexService = new SearchIndexService(mockDBConnection); + + const mockFeaturePropertyTypes = [ + { feature_property_type_id: 1, name: 'string' }, + { feature_property_type_id: 2, name: 'number' }, + { feature_property_type_id: 3, name: 'datetime' }, + { feature_property_type_id: 4, name: 'spatial' }, + { feature_property_type_id: 5, name: 'boolean' }, + { feature_property_type_id: 6, name: 'object' }, + { feature_property_type_id: 7, name: 'array' } + ] + + const mockFeatureTypes = [ + { + feature_type_id: 1, + name: 'dataset' + }, + { + feature_type_id: 2, + name: 'observation' + } + ] + + + const getFeaturesStub = sinon + .stub(SubmissionRepository.prototype, 'getSubmissionFeaturesBySubmissionId') + .resolves([ + { + submission_feature_id: 1, + submission_id: 1, // Mock submission + feature_type_id: 1, // dataset + data: { + // + } + } + ]) + + }); + }); +}); From 79b895cef4fb0b86229ce48ab385a6ba1dc8614f Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 12 Dec 2023 21:12:21 -0800 Subject: [PATCH 13/37] SIMSBIOHUB-379: Tests WIP --- .../search-index-repository.test.ts | 190 +++++++++++++++++- .../repositories/search-index-respository.ts | 18 ++ api/src/services/search-index-service.ts | 2 - 3 files changed, 207 insertions(+), 3 deletions(-) diff --git a/api/src/repositories/search-index-repository.test.ts b/api/src/repositories/search-index-repository.test.ts index a93de9ce0..b054c09a9 100644 --- a/api/src/repositories/search-index-repository.test.ts +++ b/api/src/repositories/search-index-repository.test.ts @@ -2,6 +2,194 @@ describe('SearchIndexRepository', () => { describe('getFeaturePropertiesWithTypeNames', () => { - // + const response = [ + { + property_name: 'count', + property_type: 'number', + feature_property_id: 8, + feature_property_type_id: 2, + name: 'count', + display_name: 'Count', + description: 'The count of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'date_range', + property_type: 'object', + feature_property_id: 4, + feature_property_type_id: 6, + name: 'date_range', + display_name: 'Date Range', + description: 'A date range', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'description', + property_type: 'string', + feature_property_id: 2, + feature_property_type_id: 1, + name: 'description', + display_name: 'Description', + description: 'The description of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'end_date', + property_type: 'datetime', + feature_property_id: 6, + feature_property_type_id: 3, + name: 'end_date', + display_name: 'End Date', + description: 'The end date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'geometry', + property_type: 'spatial', + feature_property_id: 7, + feature_property_type_id: 4, + name: 'geometry', + display_name: 'Geometry', + description: 'The location of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'latitude', + property_type: 'number', + feature_property_id: 9, + feature_property_type_id: 2, + name: 'latitude', + display_name: 'Latitude', + description: 'The latitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'longitude', + property_type: 'number', + feature_property_id: 10, + feature_property_type_id: 2, + name: 'longitude', + display_name: 'Longitude', + description: 'The longitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'name', + property_type: 'string', + feature_property_id: 1, + feature_property_type_id: 1, + name: 'name', + display_name: 'Name', + description: 'The name of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 's3_key', + property_type: 'string', + feature_property_id: 21, + feature_property_type_id: 1, + name: 's3_key', + display_name: 'Key', + description: 'The S3 storage key for an artifact', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 15:40:29.486362-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'start_date', + property_type: 'datetime', + feature_property_id: 5, + feature_property_type_id: 3, + name: 'start_date', + display_name: 'Start Date', + description: 'The start date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + property_name: 'taxonomy', + property_type: 'number', + feature_property_id: 3, + feature_property_type_id: 2, + name: 'taxonomy', + display_name: 'Taxonomy Id', + description: 'The taxonomy Id associated to the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + } + ] }) }) diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index 5000e3aae..eeb43ecf4 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -7,6 +7,24 @@ import SQL from "sql-template-strings"; const defaultLog = getLogger('repositories/search-index-repository'); +const FeaturePropertyRecord = z.object({ + name: z.string(), + feature_property_id: z.number(), + feature_property_type_id: z.number(), + display_name: z.string(), + description: z.string(), + parent_feature_property_id: z.number().nullable(), + record_effective_date: z.string(), + record_end_date: z.string().nullable(), + create_date: z.string(), + create_user: z.number(), + update_date: z.string().nullable(), + update_user: z.string().nullable(), + revision_count: z.number() +}); + +export type FeaturePropertyRecord = z.infer; + // TODO replace with pre-existing Zod types for geojson const Geometry = z.object({ type: z.literal('Point'), diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 9e2027dff..7c278c3ec 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -19,8 +19,6 @@ export class SearchIndexService extends DBService { const submissionRepository = new SubmissionRepository(this.connection); const features = await submissionRepository.getSubmissionFeaturesBySubmissionId(submissionId); - defaultLog.debug({ features }) - const datetimeRecords: InsertDatetimeSearchableRecord[] = []; const numberRecords: InsertNumberSearchableRecord[] = []; const spatialRecords: InsertSpatialSearchableRecord[] = []; From 0d3ced35ccd961781753bb057e1465f3852ca152 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Wed, 13 Dec 2023 13:07:31 -0800 Subject: [PATCH 14/37] SIMSBIOHUB-379: Refined types, jsdocs --- .../search-index-repository.test.ts | 22 ++++++------ .../repositories/search-index-respository.ts | 24 +++++++++---- api/src/repositories/submission-repository.ts | 12 ------- api/src/services/search-index-service.ts | 34 +++++++++++-------- 4 files changed, 49 insertions(+), 43 deletions(-) diff --git a/api/src/repositories/search-index-repository.test.ts b/api/src/repositories/search-index-repository.test.ts index b054c09a9..9fe6f26be 100644 --- a/api/src/repositories/search-index-repository.test.ts +++ b/api/src/repositories/search-index-repository.test.ts @@ -4,7 +4,7 @@ describe('SearchIndexRepository', () => { describe('getFeaturePropertiesWithTypeNames', () => { const response = [ { - property_name: 'count', + feature_property_type_name: 'count', property_type: 'number', feature_property_id: 8, feature_property_type_id: 2, @@ -21,7 +21,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'date_range', + feature_property_type_name: 'date_range', property_type: 'object', feature_property_id: 4, feature_property_type_id: 6, @@ -38,7 +38,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'description', + feature_property_type_name: 'description', property_type: 'string', feature_property_id: 2, feature_property_type_id: 1, @@ -55,7 +55,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'end_date', + feature_property_type_name: 'end_date', property_type: 'datetime', feature_property_id: 6, feature_property_type_id: 3, @@ -72,7 +72,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'geometry', + feature_property_type_name: 'geometry', property_type: 'spatial', feature_property_id: 7, feature_property_type_id: 4, @@ -89,7 +89,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'latitude', + feature_property_type_name: 'latitude', property_type: 'number', feature_property_id: 9, feature_property_type_id: 2, @@ -106,7 +106,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'longitude', + feature_property_type_name: 'longitude', property_type: 'number', feature_property_id: 10, feature_property_type_id: 2, @@ -123,7 +123,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'name', + feature_property_type_name: 'name', property_type: 'string', feature_property_id: 1, feature_property_type_id: 1, @@ -140,7 +140,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 's3_key', + feature_property_type_name: 's3_key', property_type: 'string', feature_property_id: 21, feature_property_type_id: 1, @@ -157,7 +157,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'start_date', + feature_property_type_name: 'start_date', property_type: 'datetime', feature_property_id: 5, feature_property_type_id: 3, @@ -174,7 +174,7 @@ describe('SearchIndexRepository', () => { revision_count: 0 }, { - property_name: 'taxonomy', + feature_property_type_name: 'taxonomy', property_type: 'number', feature_property_id: 3, feature_property_type_id: 2, diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index eeb43ecf4..f010459ff 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -8,9 +8,9 @@ import SQL from "sql-template-strings"; const defaultLog = getLogger('repositories/search-index-repository'); const FeaturePropertyRecord = z.object({ - name: z.string(), feature_property_id: z.number(), feature_property_type_id: z.number(), + name: z.string(), display_name: z.string(), description: z.string(), parent_feature_property_id: z.number().nullable(), @@ -25,6 +25,13 @@ const FeaturePropertyRecord = z.object({ export type FeaturePropertyRecord = z.infer; +const FeaturePropertyRecordWithPropertyTypeName = FeaturePropertyRecord.extend({ + feature_property_type_name: z.string() +}); + +export type FeaturePropertyRecordWithPropertyTypeName = z.infer; + + // TODO replace with pre-existing Zod types for geojson const Geometry = z.object({ type: z.literal('Point'), @@ -193,12 +200,17 @@ export class SearchIndexRepository extends BaseRepository { return response.rows; } - // TODO return type - async getFeaturePropertiesWithTypeNames(): Promise { + /** + * Retrieves all feature properties, with each property's type name (e.g. string, datetime, number) joined + * to it. + * + * @return {*} {Promise} + * @memberof SearchIndexRepository + */ + async getFeaturePropertiesWithTypeNames(): Promise { const query = SQL` SELECT - fp.name as property_name, - fpt.name as property_type, + fpt.name as feature_property_type_name, fp.* FROM feature_property fp @@ -212,7 +224,7 @@ export class SearchIndexRepository extends BaseRepository { fpt.record_end_date IS NULL `; - const response = await this.connection.sql(query); + const response = await this.connection.sql(query, FeaturePropertyRecordWithPropertyTypeName); return response.rows; } diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts index 421a3b8d2..7a1abeb30 100644 --- a/api/src/repositories/submission-repository.ts +++ b/api/src/repositories/submission-repository.ts @@ -362,18 +362,6 @@ export class SubmissionRepository extends BaseRepository { return response.rows[0]; } - // TODO probably safe to remove? Duplicate of getSubmissionFeaturesBySubmissionId() - // async getFeatureRecordsBySubmissionId(submissionId: number): Promise { - // const queryBuilder = getKnex() - // .select('*') - // .from('submission_feature') - // .where('submission_id', submissionId); - - // const response = await this.connection.knex(queryBuilder); - - // return response.rows; - // } - /** * Get feature type id by name. * diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 7c278c3ec..8a61d17c5 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -1,5 +1,5 @@ import { IDBConnection } from "../database/db"; -import { Geometry, InsertDatetimeSearchableRecord, InsertNumberSearchableRecord, InsertSpatialSearchableRecord, InsertStringSearchableRecord, SearchIndexRepository } from "../repositories/search-index-respository"; +import { FeaturePropertyRecord, FeaturePropertyRecordWithPropertyTypeName, Geometry, InsertDatetimeSearchableRecord, InsertNumberSearchableRecord, InsertSpatialSearchableRecord, InsertStringSearchableRecord, SearchIndexRepository } from "../repositories/search-index-respository"; import { SubmissionRepository } from "../repositories/submission-repository"; import { getLogger } from "../utils/logger"; import { DBService } from "./db-service"; @@ -15,37 +15,44 @@ export class SearchIndexService extends DBService { this.searchIndexRepository = new SearchIndexRepository(connection); } + /** + * Creates search indexes for datetime, number, spatial and string properties belonging to + * all features found for the given submission. + * + * @param {number} submissionId + * @return {*} {Promise} + * @memberof SearchIndexService + */ async indexFeaturesBySubmissionId(submissionId: number): Promise { - const submissionRepository = new SubmissionRepository(this.connection); - const features = await submissionRepository.getSubmissionFeaturesBySubmissionId(submissionId); + defaultLog.debug({ label: 'indexFeaturesBySubmissionId' }); const datetimeRecords: InsertDatetimeSearchableRecord[] = []; const numberRecords: InsertNumberSearchableRecord[] = []; const spatialRecords: InsertSpatialSearchableRecord[] = []; const stringRecords: InsertStringSearchableRecord[] = []; - const featurePropertyTypeNames = await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); + const submissionRepository = new SubmissionRepository(this.connection); + const features = await submissionRepository.getSubmissionFeaturesBySubmissionId(submissionId); - const featurePropertyTypeMap = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { - const { property_name, ...rest } = propertyType; - return [property_name, rest]; + const featurePropertyTypeNames: FeaturePropertyRecordWithPropertyTypeName[] = await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); + const featurePropertyTypeMap: Record = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { + const { feature_property_type_name, ...rest } = propertyType; + return [feature_property_type_name, rest]; })) - defaultLog.debug({ featurePropertyTypeMap }) - features.forEach((feature) => { const { submission_feature_id } = feature; Object .entries(feature.data.properties) - .forEach(([property_name, value]) => { - const featureProperty = featurePropertyTypeMap[property_name]; + .forEach(([feature_property_name, value]) => { + const featureProperty = featurePropertyTypeMap[feature_property_name]; if (!featureProperty) { return; } - const { property_type, feature_property_id } = featureProperty; + const { name, feature_property_id } = featureProperty; - switch (property_type) { + switch (name) { case 'datetime': datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); break; @@ -65,7 +72,6 @@ export class SearchIndexService extends DBService { }) }); - if (datetimeRecords.length) { this.searchIndexRepository.insertSearchableDatetimeRecords(datetimeRecords); } From 619538f03550490e2cf709441315b510df9e1da5 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Wed, 13 Dec 2023 13:14:21 -0800 Subject: [PATCH 15/37] SIMSBIOHUB-379: Added repo tests --- .../search-index-repository.test.ts | 578 ++++++++++++------ 1 file changed, 386 insertions(+), 192 deletions(-) diff --git a/api/src/repositories/search-index-repository.test.ts b/api/src/repositories/search-index-repository.test.ts index 9fe6f26be..0cefaf19e 100644 --- a/api/src/repositories/search-index-repository.test.ts +++ b/api/src/repositories/search-index-repository.test.ts @@ -1,195 +1,389 @@ - +import { QueryResult } from "pg"; +import { FeaturePropertyRecordWithPropertyTypeName, SearchIndexRepository } from "./search-index-respository"; +import { getMockDBConnection } from "../__mocks__/db"; +import { expect } from "chai"; +import Sinon from "sinon"; describe('SearchIndexRepository', () => { - describe('getFeaturePropertiesWithTypeNames', () => { - const response = [ - { - feature_property_type_name: 'count', - property_type: 'number', - feature_property_id: 8, - feature_property_type_id: 2, - name: 'count', - display_name: 'Count', - description: 'The count of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'date_range', - property_type: 'object', - feature_property_id: 4, - feature_property_type_id: 6, - name: 'date_range', - display_name: 'Date Range', - description: 'A date range', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'description', - property_type: 'string', - feature_property_id: 2, - feature_property_type_id: 1, - name: 'description', - display_name: 'Description', - description: 'The description of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'end_date', - property_type: 'datetime', - feature_property_id: 6, - feature_property_type_id: 3, - name: 'end_date', - display_name: 'End Date', - description: 'The end date of the record', - parent_feature_property_id: 4, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'geometry', - property_type: 'spatial', - feature_property_id: 7, - feature_property_type_id: 4, - name: 'geometry', - display_name: 'Geometry', - description: 'The location of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'latitude', - property_type: 'number', - feature_property_id: 9, - feature_property_type_id: 2, - name: 'latitude', - display_name: 'Latitude', - description: 'The latitude of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'longitude', - property_type: 'number', - feature_property_id: 10, - feature_property_type_id: 2, - name: 'longitude', - display_name: 'Longitude', - description: 'The longitude of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'name', - property_type: 'string', - feature_property_id: 1, - feature_property_type_id: 1, - name: 'name', - display_name: 'Name', - description: 'The name of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 's3_key', - property_type: 'string', - feature_property_id: 21, - feature_property_type_id: 1, - name: 's3_key', - display_name: 'Key', - description: 'The S3 storage key for an artifact', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 15:40:29.486362-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'start_date', - property_type: 'datetime', - feature_property_id: 5, - feature_property_type_id: 3, - name: 'start_date', - display_name: 'Start Date', - description: 'The start date of the record', - parent_feature_property_id: 4, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'taxonomy', - property_type: 'number', - feature_property_id: 3, - feature_property_type_id: 2, - name: 'taxonomy', - display_name: 'Taxonomy Id', - description: 'The taxonomy Id associated to the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - } - ] - }) + afterEach(() => { + Sinon.restore(); + }); + + describe.only('getFeaturePropertiesWithTypeNames', () => { + + it('returns an array of FeaturePropertyRecordWithPropertyTypeName', async () => { + const rows: FeaturePropertyRecordWithPropertyTypeName[] = [ + { + feature_property_type_name: 'number', + feature_property_id: 8, + feature_property_type_id: 2, + name: 'count', + display_name: 'Count', + description: 'The count of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'object', + feature_property_id: 4, + feature_property_type_id: 6, + name: 'date_range', + display_name: 'Date Range', + description: 'A date range', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 2, + feature_property_type_id: 1, + name: 'description', + display_name: 'Description', + description: 'The description of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 6, + feature_property_type_id: 3, + name: 'end_date', + display_name: 'End Date', + description: 'The end date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'spatial', + feature_property_id: 7, + feature_property_type_id: 4, + name: 'geometry', + display_name: 'Geometry', + description: 'The location of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 9, + feature_property_type_id: 2, + name: 'latitude', + display_name: 'Latitude', + description: 'The latitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 10, + feature_property_type_id: 2, + name: 'longitude', + display_name: 'Longitude', + description: 'The longitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 1, + feature_property_type_id: 1, + name: 'name', + display_name: 'Name', + description: 'The name of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 21, + feature_property_type_id: 1, + name: 's3_key', + display_name: 'Key', + description: 'The S3 storage key for an artifact', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 15:40:29.486362-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 5, + feature_property_type_id: 3, + name: 'start_date', + display_name: 'Start Date', + description: 'The start date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 3, + feature_property_type_id: 2, + name: 'taxonomy', + display_name: 'Taxonomy Id', + description: 'The taxonomy Id associated to the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + } + ]; + + const mockQueryResponse = { + rowCount: 1, + rows + } as any as Promise>; + + const mockDBConnection = getMockDBConnection({ + sql: async () => { + return mockQueryResponse; + } + }); + + const searchIndexRepository = new SearchIndexRepository(mockDBConnection); + + const response = await searchIndexRepository.getFeaturePropertiesWithTypeNames(); + + expect(response).to.eql([ + { + feature_property_type_name: 'number', + feature_property_id: 8, + feature_property_type_id: 2, + name: 'count', + display_name: 'Count', + description: 'The count of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'object', + feature_property_id: 4, + feature_property_type_id: 6, + name: 'date_range', + display_name: 'Date Range', + description: 'A date range', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 2, + feature_property_type_id: 1, + name: 'description', + display_name: 'Description', + description: 'The description of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 6, + feature_property_type_id: 3, + name: 'end_date', + display_name: 'End Date', + description: 'The end date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'spatial', + feature_property_id: 7, + feature_property_type_id: 4, + name: 'geometry', + display_name: 'Geometry', + description: 'The location of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 9, + feature_property_type_id: 2, + name: 'latitude', + display_name: 'Latitude', + description: 'The latitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 10, + feature_property_type_id: 2, + name: 'longitude', + display_name: 'Longitude', + description: 'The longitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 1, + feature_property_type_id: 1, + name: 'name', + display_name: 'Name', + description: 'The name of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 21, + feature_property_type_id: 1, + name: 's3_key', + display_name: 'Key', + description: 'The S3 storage key for an artifact', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 15:40:29.486362-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 5, + feature_property_type_id: 3, + name: 'start_date', + display_name: 'Start Date', + description: 'The start date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 3, + feature_property_type_id: 2, + name: 'taxonomy', + display_name: 'Taxonomy Id', + description: 'The taxonomy Id associated to the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + } + ]); + }); + }); }) From 57917bb7cc9b4e9965aa1049c0799878df252f2e Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Wed, 13 Dec 2023 14:06:03 -0800 Subject: [PATCH 16/37] SIMSBIOHUB-379: Working tests --- .../search-index-repository.test.ts | 2 +- api/src/services/search-index-service.test.ts | 363 ++++++++++++++++-- api/src/services/search-index-service.ts | 26 +- 3 files changed, 355 insertions(+), 36 deletions(-) diff --git a/api/src/repositories/search-index-repository.test.ts b/api/src/repositories/search-index-repository.test.ts index 0cefaf19e..8b63b61a5 100644 --- a/api/src/repositories/search-index-repository.test.ts +++ b/api/src/repositories/search-index-repository.test.ts @@ -9,7 +9,7 @@ describe('SearchIndexRepository', () => { Sinon.restore(); }); - describe.only('getFeaturePropertiesWithTypeNames', () => { + describe('getFeaturePropertiesWithTypeNames', () => { it('returns an array of FeaturePropertyRecordWithPropertyTypeName', async () => { const rows: FeaturePropertyRecordWithPropertyTypeName[] = [ diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts index 26d0e8fa7..7bb49cb29 100644 --- a/api/src/services/search-index-service.test.ts +++ b/api/src/services/search-index-service.test.ts @@ -9,7 +9,7 @@ import { SubmissionRepository } from '../repositories/submission-repository'; chai.use(sinonChai); -describe('SearchIndexService', () => { +describe.only('SearchIndexService', () => { afterEach(() => { sinon.restore(); }); @@ -20,41 +20,352 @@ describe('SearchIndexService', () => { const searchIndexService = new SearchIndexService(mockDBConnection); - const mockFeaturePropertyTypes = [ - { feature_property_type_id: 1, name: 'string' }, - { feature_property_type_id: 2, name: 'number' }, - { feature_property_type_id: 3, name: 'datetime' }, - { feature_property_type_id: 4, name: 'spatial' }, - { feature_property_type_id: 5, name: 'boolean' }, - { feature_property_type_id: 6, name: 'object' }, - { feature_property_type_id: 7, name: 'array' } - ] + // TODO probably not needed... only the properties of the JSON blob gte parsed out (and not the id or type) + // const mockFeatureTypes = [ + // { + // feature_type_id: 1, + // name: 'dataset' + // }, + // { + // feature_type_id: 2, + // name: 'observation' + // } + // ] - const mockFeatureTypes = [ - { - feature_type_id: 1, - name: 'dataset' - }, - { - feature_type_id: 2, - name: 'observation' - } - ] - - - const getFeaturesStub = sinon + const getSubmissionFeaturesStub = sinon .stub(SubmissionRepository.prototype, 'getSubmissionFeaturesBySubmissionId') .resolves([ { - submission_feature_id: 1, + submission_feature_id: 11111, submission_id: 1, // Mock submission - feature_type_id: 1, // dataset + feature_type_id: 1, // dataset, observation, whatever. data: { - // + id: 100, + type: 'some_random_thing', + properties: { + name: 'Ardvark', + description: 'Desc1', + taxonomy: 1001, + start_date: new Date('2000-01-01'), + geometry: { type: 'Point', coordinates: [11, 11] }, + count: 60, + latitude: 11, + longitude: 11 + } + } + }, + { + submission_feature_id: 22222, + submission_id: 1, // Mock submission + feature_type_id: 1, // dataset, observation, whatever. + data: { + id: 200, + type: 'another_random_thing', + properties: { + name: 'Buffalo', + description: 'Desc2', + taxonomy: 1002, + start_date: new Date('2001-01-01'), + geometry: { type: 'Point', coordinates: [22, 22] }, + count: 70, + latitude: 22, + longitude: 22 + } } } ]) + const insertSearchableStringStub = sinon + .stub(SearchIndexRepository.prototype, 'insertSearchableStringRecords') + + const insertSearchableDatetimeStub = sinon + .stub(SearchIndexRepository.prototype, 'insertSearchableDatetimeRecords') + + const insertSearchableSpatialStub = sinon + .stub(SearchIndexRepository.prototype, 'insertSearchableSpatialRecords') + + const insertSearchableNumberStub = sinon + .stub(SearchIndexRepository.prototype, 'insertSearchableNumberRecords') + + /*const getFeaturePropertiesWithTypeNamesStub = */ // TODO remove? + sinon.stub(SearchIndexRepository.prototype, 'getFeaturePropertiesWithTypeNames') + .resolves([ + { + feature_property_type_name: 'number', + feature_property_id: 8, + feature_property_type_id: 2, + name: 'count', + display_name: 'Count', + description: 'The count of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'object', + feature_property_id: 4, + feature_property_type_id: 6, + name: 'date_range', + display_name: 'Date Range', + description: 'A date range', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 2, + feature_property_type_id: 1, + name: 'description', + display_name: 'Description', + description: 'The description of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 6, + feature_property_type_id: 3, + name: 'end_date', + display_name: 'End Date', + description: 'The end date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'spatial', + feature_property_id: 7, + feature_property_type_id: 4, + name: 'geometry', + display_name: 'Geometry', + description: 'The location of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 9, + feature_property_type_id: 2, + name: 'latitude', + display_name: 'Latitude', + description: 'The latitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 10, + feature_property_type_id: 2, + name: 'longitude', + display_name: 'Longitude', + description: 'The longitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 1, + feature_property_type_id: 1, + name: 'name', + display_name: 'Name', + description: 'The name of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 21, + feature_property_type_id: 1, + name: 's3_key', + display_name: 'Key', + description: 'The S3 storage key for an artifact', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 15:40:29.486362-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 5, + feature_property_type_id: 3, + name: 'start_date', + display_name: 'Start Date', + description: 'The start date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 3, + feature_property_type_id: 2, + name: 'taxonomy', + display_name: 'Taxonomy Id', + description: 'The taxonomy Id associated to the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + } + ]); + + // Act + await searchIndexService.indexFeaturesBySubmissionId(777); + + // Assert + expect(getSubmissionFeaturesStub).to.be.calledWith(777); + + expect(insertSearchableStringStub).to.be.calledWith([ + { + submission_feature_id: 11111, + feature_property_id: 1, // Name + value: 'Ardvark' + }, + { + submission_feature_id: 11111, + feature_property_id: 2, // Description + value: 'Desc1' + }, + { + submission_feature_id: 22222, + feature_property_id: 1, // Name + value: 'Buffalo' + }, + { + submission_feature_id: 22222, + feature_property_id: 2, // Description + value: 'Desc2' + } + ]); + + expect(insertSearchableDatetimeStub).to.be.calledWith([ + { + submission_feature_id: 11111, + feature_property_id: 5, // Start Date + value: new Date('2000-01-01') + }, + { + submission_feature_id: 22222, + feature_property_id: 5, // Start Date + value: new Date('2001-01-01') + } + ]); + + expect(insertSearchableSpatialStub).to.be.calledWith([ + { + submission_feature_id: 11111, + feature_property_id: 7, // Spatial + value: { type: 'Point', coordinates: [11, 11] } + }, + { + submission_feature_id: 22222, + feature_property_id: 7, // Spatial + value: { type: 'Point', coordinates: [22, 22] } + } + ]); + + expect(insertSearchableNumberStub).to.be.calledWith([ + { + submission_feature_id: 11111, + feature_property_id: 3, // Taxonomy + value: 1001 + }, + { + submission_feature_id: 11111, + feature_property_id: 8, // Count + value: 60 + }, + { + submission_feature_id: 11111, + feature_property_id: 9, // Lat + value: 11 + }, + { + submission_feature_id: 11111, + feature_property_id: 10, // Long + value: 11 + }, + { + submission_feature_id: 22222, + feature_property_id: 3, // Taxonomy + value: 1002 + }, + { + submission_feature_id: 22222, + feature_property_id: 8, // Count + value: 70 + }, + { + submission_feature_id: 22222, + feature_property_id: 9, // Lat + value: 22 + }, + { + submission_feature_id: 22222, + feature_property_id: 10, // Long + value: 22 + } + ]); + }); }); }); diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 8a61d17c5..88f64f180 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -1,5 +1,13 @@ import { IDBConnection } from "../database/db"; -import { FeaturePropertyRecord, FeaturePropertyRecordWithPropertyTypeName, Geometry, InsertDatetimeSearchableRecord, InsertNumberSearchableRecord, InsertSpatialSearchableRecord, InsertStringSearchableRecord, SearchIndexRepository } from "../repositories/search-index-respository"; +import { + FeaturePropertyRecordWithPropertyTypeName, + Geometry, + InsertDatetimeSearchableRecord, + InsertNumberSearchableRecord, + InsertSpatialSearchableRecord, + InsertStringSearchableRecord, + SearchIndexRepository +} from "../repositories/search-index-respository"; import { SubmissionRepository } from "../repositories/submission-repository"; import { getLogger } from "../utils/logger"; import { DBService } from "./db-service"; @@ -31,28 +39,28 @@ export class SearchIndexService extends DBService { const spatialRecords: InsertSpatialSearchableRecord[] = []; const stringRecords: InsertStringSearchableRecord[] = []; - const submissionRepository = new SubmissionRepository(this.connection); + const submissionRepository = new SubmissionRepository(this.connection); const features = await submissionRepository.getSubmissionFeaturesBySubmissionId(submissionId); const featurePropertyTypeNames: FeaturePropertyRecordWithPropertyTypeName[] = await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); - const featurePropertyTypeMap: Record = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { - const { feature_property_type_name, ...rest } = propertyType; - return [feature_property_type_name, rest]; + const featurePropertyTypeMap: Record = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { + const { name } = propertyType; + return [name, propertyType]; })) features.forEach((feature) => { const { submission_feature_id } = feature; Object .entries(feature.data.properties) - .forEach(([feature_property_name, value]) => { + .forEach(([feature_property_name, value]) => { const featureProperty = featurePropertyTypeMap[feature_property_name]; if (!featureProperty) { return; } - const { name, feature_property_id } = featureProperty; + const { feature_property_type_name, feature_property_id } = featureProperty; - switch (name) { + switch (feature_property_type_name) { case 'datetime': datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); break; @@ -70,7 +78,7 @@ export class SearchIndexService extends DBService { break; } }) - }); + }); if (datetimeRecords.length) { this.searchIndexRepository.insertSearchableDatetimeRecords(datetimeRecords); From 97b20f0844772b50bf9610a18904523958f0e6b7 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Wed, 13 Dec 2023 14:15:34 -0800 Subject: [PATCH 17/37] SIMSBIOHUB-379: Code cleanup --- api/src/paths/submission/search-idx.ts | 7 +- .../search-index-repository.test.ts | 13 +- .../repositories/search-index-respository.ts | 68 ++- api/src/services/search-index-service.test.ts | 398 +++++++++--------- api/src/services/search-index-service.ts | 67 +-- app/src/features/home/HomePage.tsx | 3 - app/src/hooks/api/useSubmissionsApi.ts | 10 +- 7 files changed, 264 insertions(+), 302 deletions(-) diff --git a/api/src/paths/submission/search-idx.ts b/api/src/paths/submission/search-idx.ts index 3ad46d0c5..6be7bc8d5 100644 --- a/api/src/paths/submission/search-idx.ts +++ b/api/src/paths/submission/search-idx.ts @@ -4,8 +4,8 @@ import { SOURCE_SYSTEM } from '../../constants/database'; import { getAPIUserDBConnection, getDBConnection } from '../../database/db'; import { defaultErrorResponses } from '../../openapi/schemas/http-responses'; import { authorizeRequestHandler } from '../../request-handlers/security/authorization'; -import { getLogger } from '../../utils/logger'; import { SearchIndexService } from '../../services/search-index-service'; +import { getLogger } from '../../utils/logger'; const defaultLog = getLogger('paths/dataset/search-index'); @@ -60,7 +60,6 @@ POST.apiDoc = { export function indexSubmission(): RequestHandler { return async (req, res) => { - const connection = req['keycloak_token'] ? getDBConnection(req['keycloak_token']) : getAPIUserDBConnection(); const submissionId = Number(req.query.submissionId); @@ -69,9 +68,9 @@ export function indexSubmission(): RequestHandler { await connection.open(); const searchIndexService = new SearchIndexService(connection); - + // Index the submission record - const response = await searchIndexService.indexFeaturesBySubmissionId(submissionId) + const response = await searchIndexService.indexFeaturesBySubmissionId(submissionId); await connection.commit(); res.status(200).json(response); diff --git a/api/src/repositories/search-index-repository.test.ts b/api/src/repositories/search-index-repository.test.ts index 8b63b61a5..1abfb3214 100644 --- a/api/src/repositories/search-index-repository.test.ts +++ b/api/src/repositories/search-index-repository.test.ts @@ -1,8 +1,8 @@ -import { QueryResult } from "pg"; -import { FeaturePropertyRecordWithPropertyTypeName, SearchIndexRepository } from "./search-index-respository"; -import { getMockDBConnection } from "../__mocks__/db"; -import { expect } from "chai"; -import Sinon from "sinon"; +import { expect } from 'chai'; +import { QueryResult } from 'pg'; +import Sinon from 'sinon'; +import { getMockDBConnection } from '../__mocks__/db'; +import { FeaturePropertyRecordWithPropertyTypeName, SearchIndexRepository } from './search-index-respository'; describe('SearchIndexRepository', () => { afterEach(() => { @@ -10,7 +10,6 @@ describe('SearchIndexRepository', () => { }); describe('getFeaturePropertiesWithTypeNames', () => { - it('returns an array of FeaturePropertyRecordWithPropertyTypeName', async () => { const rows: FeaturePropertyRecordWithPropertyTypeName[] = [ { @@ -386,4 +385,4 @@ describe('SearchIndexRepository', () => { ]); }); }); -}) +}); diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index f010459ff..c41fa34d4 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -1,9 +1,9 @@ -import { z } from "zod"; -import { getLogger } from "../utils/logger"; -import { BaseRepository } from "./base-repository"; -import { getKnex } from "../database/db"; -import { ApiExecuteSQLError } from "../errors/api-error"; -import SQL from "sql-template-strings"; +import SQL from 'sql-template-strings'; +import { z } from 'zod'; +import { getKnex } from '../database/db'; +import { ApiExecuteSQLError } from '../errors/api-error'; +import { getLogger } from '../utils/logger'; +import { BaseRepository } from './base-repository'; const defaultLog = getLogger('repositories/search-index-repository'); @@ -31,11 +31,10 @@ const FeaturePropertyRecordWithPropertyTypeName = FeaturePropertyRecord.extend({ export type FeaturePropertyRecordWithPropertyTypeName = z.infer; - // TODO replace with pre-existing Zod types for geojson const Geometry = z.object({ type: z.literal('Point'), - coordinates: z.tuple([z.number(), z.number()]), + coordinates: z.tuple([z.number(), z.number()]) }); export type Geometry = z.infer; @@ -48,10 +47,10 @@ const SearchableRecord = z.object({ create_user: z.number(), update_date: z.date().nullable(), update_user: z.date().nullable(), - revision_count: z.number(), + revision_count: z.number() }); -type InsertSearchableRecordKey = 'submission_feature_id' | 'value' | 'feature_property_id' +type InsertSearchableRecordKey = 'submission_feature_id' | 'value' | 'feature_property_id'; export const DatetimeSearchableRecord = SearchableRecord.extend({ search_datetime_id: z.date(), @@ -79,10 +78,10 @@ export type NumberSearchableRecord = z.infer; export type SpatialSearchableRecord = z.infer; export type StringSearchableRecord = z.infer; -export type InsertDatetimeSearchableRecord = Pick -export type InsertNumberSearchableRecord = Pick -export type InsertSpatialSearchableRecord = Pick -export type InsertStringSearchableRecord = Pick +export type InsertDatetimeSearchableRecord = Pick; +export type InsertNumberSearchableRecord = Pick; +export type InsertSpatialSearchableRecord = Pick; +export type InsertStringSearchableRecord = Pick; /** * A class for creating searchable records @@ -95,14 +94,12 @@ export class SearchIndexRepository extends BaseRepository { * @return {*} {Promise} * @memberof SearchIndexRepository */ - async insertSearchableDatetimeRecords(datetimeRecords: InsertDatetimeSearchableRecord[]): Promise { + async insertSearchableDatetimeRecords( + datetimeRecords: InsertDatetimeSearchableRecord[] + ): Promise { defaultLog.debug({ label: 'insertSearchableDatetimeRecords' }); - const queryBuilder = getKnex() - .queryBuilder() - .insert(datetimeRecords) - .into('search_datetime') - .returning('*') + const queryBuilder = getKnex().queryBuilder().insert(datetimeRecords).into('search_datetime').returning('*'); const response = await this.connection.knex(queryBuilder); @@ -123,14 +120,12 @@ export class SearchIndexRepository extends BaseRepository { * @return {*} {Promise} * @memberof SearchIndexRepository */ - async insertSearchableNumberRecords(numberRecords: InsertNumberSearchableRecord[]): Promise { + async insertSearchableNumberRecords( + numberRecords: InsertNumberSearchableRecord[] + ): Promise { defaultLog.debug({ label: 'insertSearchableNumberRecords' }); - const queryBuilder = getKnex() - .queryBuilder() - .insert(numberRecords) - .into('search_number') - .returning('*') + const queryBuilder = getKnex().queryBuilder().insert(numberRecords).into('search_number').returning('*'); const response = await this.connection.knex(queryBuilder); @@ -151,14 +146,12 @@ export class SearchIndexRepository extends BaseRepository { * @return {*} {Promise} * @memberof SearchIndexRepository */ - async insertSearchableSpatialRecords(spatialRecords: InsertSpatialSearchableRecord[]): Promise { + async insertSearchableSpatialRecords( + spatialRecords: InsertSpatialSearchableRecord[] + ): Promise { defaultLog.debug({ label: 'insertSearchableSpatialRecords' }); - const queryBuilder = getKnex() - .queryBuilder() - .insert(spatialRecords) - .into('search_spatial') - .returning('*') + const queryBuilder = getKnex().queryBuilder().insert(spatialRecords).into('search_spatial').returning('*'); const response = await this.connection.knex(queryBuilder); @@ -179,14 +172,12 @@ export class SearchIndexRepository extends BaseRepository { * @return {*} {Promise} * @memberof SearchIndexRepository */ - async insertSearchableStringRecords(stringRecords: InsertStringSearchableRecord[]): Promise { + async insertSearchableStringRecords( + stringRecords: InsertStringSearchableRecord[] + ): Promise { defaultLog.debug({ label: 'insertSearchableStringRecords' }); - const queryBuilder = getKnex() - .queryBuilder() - .insert(stringRecords) - .into('search_string') - .returning('*') + const queryBuilder = getKnex().queryBuilder().insert(stringRecords).into('search_string').returning('*'); const response = await this.connection.knex(queryBuilder); @@ -228,5 +219,4 @@ export class SearchIndexRepository extends BaseRepository { return response.rows; } - } diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts index 7bb49cb29..e96a6f7c3 100644 --- a/api/src/services/search-index-service.test.ts +++ b/api/src/services/search-index-service.test.ts @@ -2,14 +2,14 @@ import chai, { expect } from 'chai'; import { describe } from 'mocha'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import { getMockDBConnection } from '../__mocks__/db'; -import { SearchIndexService } from './search-index-service'; import { SearchIndexRepository } from '../repositories/search-index-respository'; import { SubmissionRepository } from '../repositories/submission-repository'; +import { getMockDBConnection } from '../__mocks__/db'; +import { SearchIndexService } from './search-index-service'; chai.use(sinonChai); -describe.only('SearchIndexService', () => { +describe('SearchIndexService', () => { afterEach(() => { sinon.restore(); }); @@ -20,25 +20,13 @@ describe.only('SearchIndexService', () => { const searchIndexService = new SearchIndexService(mockDBConnection); - // TODO probably not needed... only the properties of the JSON blob gte parsed out (and not the id or type) - // const mockFeatureTypes = [ - // { - // feature_type_id: 1, - // name: 'dataset' - // }, - // { - // feature_type_id: 2, - // name: 'observation' - // } - // ] - const getSubmissionFeaturesStub = sinon .stub(SubmissionRepository.prototype, 'getSubmissionFeaturesBySubmissionId') .resolves([ { submission_feature_id: 11111, submission_id: 1, // Mock submission - feature_type_id: 1, // dataset, observation, whatever. + feature_type_id: 1, // dataset, observation, etc. data: { id: 100, type: 'some_random_thing', @@ -57,7 +45,7 @@ describe.only('SearchIndexService', () => { { submission_feature_id: 22222, submission_id: 1, // Mock submission - feature_type_id: 1, // dataset, observation, whatever. + feature_type_id: 1, // dataset, observation, etc. data: { id: 200, type: 'another_random_thing', @@ -73,200 +61,197 @@ describe.only('SearchIndexService', () => { } } } - ]) + ]); - const insertSearchableStringStub = sinon - .stub(SearchIndexRepository.prototype, 'insertSearchableStringRecords') + const insertSearchableStringStub = sinon.stub(SearchIndexRepository.prototype, 'insertSearchableStringRecords'); - const insertSearchableDatetimeStub = sinon - .stub(SearchIndexRepository.prototype, 'insertSearchableDatetimeRecords') + const insertSearchableDatetimeStub = sinon.stub( + SearchIndexRepository.prototype, + 'insertSearchableDatetimeRecords' + ); - const insertSearchableSpatialStub = sinon - .stub(SearchIndexRepository.prototype, 'insertSearchableSpatialRecords') + const insertSearchableSpatialStub = sinon.stub(SearchIndexRepository.prototype, 'insertSearchableSpatialRecords'); - const insertSearchableNumberStub = sinon - .stub(SearchIndexRepository.prototype, 'insertSearchableNumberRecords') + const insertSearchableNumberStub = sinon.stub(SearchIndexRepository.prototype, 'insertSearchableNumberRecords'); - /*const getFeaturePropertiesWithTypeNamesStub = */ // TODO remove? - sinon.stub(SearchIndexRepository.prototype, 'getFeaturePropertiesWithTypeNames') - .resolves([ - { - feature_property_type_name: 'number', - feature_property_id: 8, - feature_property_type_id: 2, - name: 'count', - display_name: 'Count', - description: 'The count of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'object', - feature_property_id: 4, - feature_property_type_id: 6, - name: 'date_range', - display_name: 'Date Range', - description: 'A date range', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'string', - feature_property_id: 2, - feature_property_type_id: 1, - name: 'description', - display_name: 'Description', - description: 'The description of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'datetime', - feature_property_id: 6, - feature_property_type_id: 3, - name: 'end_date', - display_name: 'End Date', - description: 'The end date of the record', - parent_feature_property_id: 4, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'spatial', - feature_property_id: 7, - feature_property_type_id: 4, - name: 'geometry', - display_name: 'Geometry', - description: 'The location of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'number', - feature_property_id: 9, - feature_property_type_id: 2, - name: 'latitude', - display_name: 'Latitude', - description: 'The latitude of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'number', - feature_property_id: 10, - feature_property_type_id: 2, - name: 'longitude', - display_name: 'Longitude', - description: 'The longitude of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'string', - feature_property_id: 1, - feature_property_type_id: 1, - name: 'name', - display_name: 'Name', - description: 'The name of the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'string', - feature_property_id: 21, - feature_property_type_id: 1, - name: 's3_key', - display_name: 'Key', - description: 'The S3 storage key for an artifact', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 15:40:29.486362-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'datetime', - feature_property_id: 5, - feature_property_type_id: 3, - name: 'start_date', - display_name: 'Start Date', - description: 'The start date of the record', - parent_feature_property_id: 4, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - }, - { - feature_property_type_name: 'number', - feature_property_id: 3, - feature_property_type_id: 2, - name: 'taxonomy', - display_name: 'Taxonomy Id', - description: 'The taxonomy Id associated to the record', - parent_feature_property_id: null, - record_effective_date: '2023-12-08', - record_end_date: null, - create_date: '2023-12-08 14:37:41.315999-08', - create_user: 1, - update_date: null, - update_user: null, - revision_count: 0 - } - ]); + sinon.stub(SearchIndexRepository.prototype, 'getFeaturePropertiesWithTypeNames').resolves([ + { + feature_property_type_name: 'number', + feature_property_id: 8, + feature_property_type_id: 2, + name: 'count', + display_name: 'Count', + description: 'The count of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'object', + feature_property_id: 4, + feature_property_type_id: 6, + name: 'date_range', + display_name: 'Date Range', + description: 'A date range', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 2, + feature_property_type_id: 1, + name: 'description', + display_name: 'Description', + description: 'The description of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 6, + feature_property_type_id: 3, + name: 'end_date', + display_name: 'End Date', + description: 'The end date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'spatial', + feature_property_id: 7, + feature_property_type_id: 4, + name: 'geometry', + display_name: 'Geometry', + description: 'The location of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 9, + feature_property_type_id: 2, + name: 'latitude', + display_name: 'Latitude', + description: 'The latitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 10, + feature_property_type_id: 2, + name: 'longitude', + display_name: 'Longitude', + description: 'The longitude of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 1, + feature_property_type_id: 1, + name: 'name', + display_name: 'Name', + description: 'The name of the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'string', + feature_property_id: 21, + feature_property_type_id: 1, + name: 's3_key', + display_name: 'Key', + description: 'The S3 storage key for an artifact', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 15:40:29.486362-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'datetime', + feature_property_id: 5, + feature_property_type_id: 3, + name: 'start_date', + display_name: 'Start Date', + description: 'The start date of the record', + parent_feature_property_id: 4, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + }, + { + feature_property_type_name: 'number', + feature_property_id: 3, + feature_property_type_id: 2, + name: 'taxonomy', + display_name: 'Taxonomy Id', + description: 'The taxonomy Id associated to the record', + parent_feature_property_id: null, + record_effective_date: '2023-12-08', + record_end_date: null, + create_date: '2023-12-08 14:37:41.315999-08', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 0 + } + ]); // Act await searchIndexService.indexFeaturesBySubmissionId(777); @@ -365,7 +350,6 @@ describe.only('SearchIndexService', () => { value: 22 } ]); - }); }); }); diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 88f64f180..d8d1318ca 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -1,4 +1,4 @@ -import { IDBConnection } from "../database/db"; +import { IDBConnection } from '../database/db'; import { FeaturePropertyRecordWithPropertyTypeName, Geometry, @@ -7,10 +7,10 @@ import { InsertSpatialSearchableRecord, InsertStringSearchableRecord, SearchIndexRepository -} from "../repositories/search-index-respository"; -import { SubmissionRepository } from "../repositories/submission-repository"; -import { getLogger } from "../utils/logger"; -import { DBService } from "./db-service"; +} from '../repositories/search-index-respository'; +import { SubmissionRepository } from '../repositories/submission-repository'; +import { getLogger } from '../utils/logger'; +import { DBService } from './db-service'; const defaultLog = getLogger('services/search-index-service'); @@ -42,42 +42,43 @@ export class SearchIndexService extends DBService { const submissionRepository = new SubmissionRepository(this.connection); const features = await submissionRepository.getSubmissionFeaturesBySubmissionId(submissionId); - const featurePropertyTypeNames: FeaturePropertyRecordWithPropertyTypeName[] = await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); - const featurePropertyTypeMap: Record = Object.fromEntries(featurePropertyTypeNames.map((propertyType) => { - const { name } = propertyType; - return [name, propertyType]; - })) + const featurePropertyTypeNames: FeaturePropertyRecordWithPropertyTypeName[] = + await this.searchIndexRepository.getFeaturePropertiesWithTypeNames(); + const featurePropertyTypeMap: Record = Object.fromEntries( + featurePropertyTypeNames.map((propertyType) => { + const { name } = propertyType; + return [name, propertyType]; + }) + ); features.forEach((feature) => { const { submission_feature_id } = feature; - Object - .entries(feature.data.properties) - .forEach(([feature_property_name, value]) => { - const featureProperty = featurePropertyTypeMap[feature_property_name]; - if (!featureProperty) { - return; - } + Object.entries(feature.data.properties).forEach(([feature_property_name, value]) => { + const featureProperty = featurePropertyTypeMap[feature_property_name]; + if (!featureProperty) { + return; + } - const { feature_property_type_name, feature_property_id } = featureProperty; + const { feature_property_type_name, feature_property_id } = featureProperty; - switch (feature_property_type_name) { - case 'datetime': - datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); - break; + switch (feature_property_type_name) { + case 'datetime': + datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); + break; - case 'number': - numberRecords.push({ submission_feature_id, feature_property_id, value: value as number }); - break; + case 'number': + numberRecords.push({ submission_feature_id, feature_property_id, value: value as number }); + break; - case 'spatial': - spatialRecords.push({ submission_feature_id, feature_property_id, value: value as Geometry }); - break; + case 'spatial': + spatialRecords.push({ submission_feature_id, feature_property_id, value: value as Geometry }); + break; - case 'string': - stringRecords.push({ submission_feature_id, feature_property_id, value: value as string }); - break; - } - }) + case 'string': + stringRecords.push({ submission_feature_id, feature_property_id, value: value as string }); + break; + } + }); }); if (datetimeRecords.length) { diff --git a/app/src/features/home/HomePage.tsx b/app/src/features/home/HomePage.tsx index da7bcaf44..fd6773e81 100644 --- a/app/src/features/home/HomePage.tsx +++ b/app/src/features/home/HomePage.tsx @@ -2,7 +2,6 @@ import { Container, Paper, Typography } from '@mui/material'; import Box from '@mui/material/Box'; import SearchComponent from 'features/search/SearchComponent'; import { Formik, FormikProps } from 'formik'; -import { useApi } from 'hooks/useApi'; import { IAdvancedSearch } from 'interfaces/useSearchApi.interface'; import { useRef } from 'react'; import { useHistory } from 'react-router'; @@ -21,8 +20,6 @@ const HomePage = () => { history.push(`/search?keywords=${query}`); }; - useApi().submissions.test(1) - return ( { return data; }; - // Triggers a request to index the given submission - const test = async (submissionId: number): Promise => { - const { data } = await axios.post(`/api/submission/search-idx?submissionId=${submissionId}`); - - return data; - } - /** NET-NEW FRONTEND REQUESTS FOR UPDATED SCHEMA **/ /** @@ -94,8 +87,7 @@ const useSubmissionsApi = (axios: AxiosInstance) => { getSignedUrl, listReviewedSubmissions, getSubmissionDownloadPackage, - getSubmission, - test + getSubmission }; }; From d84e87e9c231aba64d96faf37bc26c3cd3311dfc Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Fri, 15 Dec 2023 12:09:56 -0800 Subject: [PATCH 18/37] SIMSBIOHUB-379: Fix type error --- api/src/services/search-index-service.test.ts | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts index e96a6f7c3..0885e8844 100644 --- a/api/src/services/search-index-service.test.ts +++ b/api/src/services/search-index-service.test.ts @@ -40,7 +40,18 @@ describe('SearchIndexService', () => { latitude: 11, longitude: 11 } - } + }, + parent_submission_feature_id: null, + record_effective_date: '', + record_end_date: null, + create_date: '', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 1, + feature_type_name: '', + feature_type_display_name: '', + submission_feature_security_ids: [] }, { submission_feature_id: 22222, @@ -59,7 +70,18 @@ describe('SearchIndexService', () => { latitude: 22, longitude: 22 } - } + }, + parent_submission_feature_id: null, + record_effective_date: '', + record_end_date: null, + create_date: '', + create_user: 1, + update_date: null, + update_user: null, + revision_count: 1, + feature_type_name: '', + feature_type_display_name: '', + submission_feature_security_ids: [] } ]); From f9215e3f81ff03650aee172473074d758685e16c Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 15 Dec 2023 15:43:18 -0800 Subject: [PATCH 19/37] Update migration --- database/src/migrations/20231117000001_security_tables.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/database/src/migrations/20231117000001_security_tables.ts b/database/src/migrations/20231117000001_security_tables.ts index 454bcf607..3df15c220 100644 --- a/database/src/migrations/20231117000001_security_tables.ts +++ b/database/src/migrations/20231117000001_security_tables.ts @@ -77,6 +77,7 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_string( security_string_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -93,6 +94,7 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_string.security_string_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_string.security_rule_id IS 'Foreign key to the security_string table.'; COMMENT ON COLUMN security_string.name IS 'The name of the security_string record.'; COMMENT ON COLUMN security_string.description IS 'The description of the security_string record.'; COMMENT ON COLUMN security_string.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -109,6 +111,7 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_number( security_number_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -125,6 +128,7 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_number.security_number_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_number.security_rule_id IS 'Foreign key to the security_number table.'; COMMENT ON COLUMN security_number.name IS 'The name of the security_number record.'; COMMENT ON COLUMN security_number.description IS 'The description of the security_number record.'; COMMENT ON COLUMN security_number.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -141,6 +145,7 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_datetime( security_datetime_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -157,6 +162,7 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_datetime.security_datetime_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_datetime.security_rule_id IS 'Foreign key to the security_datetime table.'; COMMENT ON COLUMN security_datetime.name IS 'The name of the security_datetime record.'; COMMENT ON COLUMN security_datetime.description IS 'The description of the security_datetime record.'; COMMENT ON COLUMN security_datetime.feature_property_id IS 'Foreign key to the feature_property table.'; @@ -173,6 +179,7 @@ export async function up(knex: Knex): Promise { CREATE TABLE security_spatial( security_spatial_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, name varchar(100) NOT NULL, description varchar(500), feature_property_id integer NOT NULL, @@ -189,6 +196,7 @@ export async function up(knex: Knex): Promise { ); COMMENT ON COLUMN security_spatial.security_spatial_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_spatial.security_rule_id IS 'Foreign key to the security_spatial table.'; COMMENT ON COLUMN security_spatial.name IS 'The name of the security_spatial record.'; COMMENT ON COLUMN security_spatial.description IS 'The description of the security_spatial record.'; COMMENT ON COLUMN security_spatial.feature_property_id IS 'Foreign key to the feature_property table.'; From 522d91edd27b9848fed64a15f2c0f6a3164a28e8 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Mon, 18 Dec 2023 11:59:49 -0800 Subject: [PATCH 20/37] Update intake endpoint to work with related changes from SIMS PR #1185 Update validation to account for intake endpoint changes (enhance to support validating any level of nested submission features) Fix bug with zod geo json schemas. Update unit tests. --- api/package-lock.json | 2 +- api/package.json | 2 +- api/src/paths/submission/intake.ts | 43 ++-- .../submission-repository.test.ts | 26 ++- api/src/repositories/submission-repository.ts | 39 ++-- api/src/services/artifact-service.ts | 6 +- api/src/services/submission-service.test.ts | 6 +- api/src/services/submission-service.ts | 18 +- api/src/services/validation-service.test.ts | 194 +++++++++++------- api/src/services/validation-service.ts | 77 ++++--- api/src/zod-schema/geoJsonZodSchema.ts | 8 +- 11 files changed, 254 insertions(+), 167 deletions(-) diff --git a/api/package-lock.json b/api/package-lock.json index 110eb451f..596921578 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -8481,7 +8481,7 @@ "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", "dev": true }, "source-map-resolve": { diff --git a/api/package.json b/api/package.json index 904b21233..ca3089e5c 100644 --- a/api/package.json +++ b/api/package.json @@ -78,10 +78,10 @@ "@types/mocha": "~9.0.0", "@types/multer": "~1.4.7", "@types/node": "~14.14.31", + "@types/object-inspect": "~1.8.1", "@types/pg": "~8.6.1", "@types/sinon": "~10.0.4", "@types/sinon-chai": "~3.2.5", - "@types/object-inspect": "~1.8.1", "@types/swagger-ui-express": "~4.1.3", "@types/uuid": "~8.3.1", "@types/yamljs": "~0.2.31", diff --git a/api/src/paths/submission/intake.ts b/api/src/paths/submission/intake.ts index 8a2eb9d1d..58a28673b 100644 --- a/api/src/paths/submission/intake.ts +++ b/api/src/paths/submission/intake.ts @@ -4,7 +4,9 @@ import { SOURCE_SYSTEM } from '../../constants/database'; import { getServiceAccountDBConnection } from '../../database/db'; import { HTTP400 } from '../../errors/http-error'; import { defaultErrorResponses } from '../../openapi/schemas/http-responses'; +import { ISubmissionFeature } from '../../repositories/submission-repository'; import { authorizeRequestHandler } from '../../request-handlers/security/authorization'; +import { SearchIndexService } from '../../services/search-index-service'; import { SubmissionService } from '../../services/submission-service'; import { ValidationService } from '../../services/validation-service'; import { getKeycloakSource } from '../../utils/keycloak-utils'; @@ -44,26 +46,28 @@ POST.apiDoc = { schema: { title: 'BioHub Data Submission', type: 'object', - required: ['id', 'type', 'properties', 'features'], + required: ['id', 'name', 'description', 'features'], properties: { id: { title: 'Unique id of the submission', type: 'string' }, - type: { + name: { + title: 'The name of the submission. Should not include sensitive information.', type: 'string', - enum: ['submission'] + maxLength: 200 }, - properties: { - title: 'Dataset properties', - type: 'object', - properties: {} + description: { + title: 'A description of the submission. Should not include sensitive information.', + type: 'string', + maxLength: 3000 }, features: { type: 'array', items: { $ref: '#/components/schemas/SubmissionFeature' }, + maxItems: 1, additionalProperties: false } }, @@ -104,10 +108,12 @@ export function submissionIntake(): RequestHandler { } const submission = { - ...req.body, - properties: { ...req.body.properties, additionalInformation: req.body.properties.additionalInformation } + id: req.body.id, + name: req.body.name, + description: req.body.description }; - const id = req.body.id; + + const submissionFeatures: ISubmissionFeature[] = req.body.features; const connection = getServiceAccountDBConnection(sourceSystem); @@ -116,19 +122,28 @@ export function submissionIntake(): RequestHandler { const submissionService = new SubmissionService(connection); const validationService = new ValidationService(connection); + const searchIndexService = new SearchIndexService(connection); - // validate the submission submission - if (!(await validationService.validateDatasetSubmission(submission))) { + // validate the submission + if (!(await validationService.validateSubmissionFeatures(submissionFeatures))) { throw new HTTP400('Invalid submission submission'); } // insert the submission record - const response = await submissionService.insertSubmissionRecordWithPotentialConflict(id); + const response = await submissionService.insertSubmissionRecordWithPotentialConflict( + submission.id, + submission.name, + submission.description + ); // insert each submission feature record - await submissionService.insertSubmissionFeatureRecords(response.submission_id, submission.features); + await submissionService.insertSubmissionFeatureRecords(response.submission_id, submissionFeatures); + + // Index the submission feature record properties + await searchIndexService.indexFeaturesBySubmissionId(response.submission_id); await connection.commit(); + res.status(200).json(response); } catch (error) { defaultLog.error({ label: 'submissionIntake', message: 'error', error }); diff --git a/api/src/repositories/submission-repository.test.ts b/api/src/repositories/submission-repository.test.ts index 7dba64b00..e8d7f9d56 100644 --- a/api/src/repositories/submission-repository.test.ts +++ b/api/src/repositories/submission-repository.test.ts @@ -481,26 +481,20 @@ describe('SubmissionRepository', () => { it('should insert or retrieve a submission successfully', async () => { const mockQueryResponse = { rowCount: 1, - rows: [ - { - uuid: 'aaaa', - source_transform_id: 1, - submission_id: 20 - } - ] + rows: [{ submission_id: 20 }] } as any as Promise>; const mockDBConnection = getMockDBConnection({ sql: async () => mockQueryResponse }); const submissionRepository = new SubmissionRepository(mockDBConnection); - const response = await submissionRepository.insertSubmissionRecordWithPotentialConflict('aaaa'); + const response = await submissionRepository.insertSubmissionRecordWithPotentialConflict( + '123-456-789', + 'submission name', + 'source system' + ); - expect(response).to.eql({ - uuid: 'aaaa', - source_transform_id: 1, - submission_id: 20 - }); + expect(response).to.eql({ submission_id: 20 }); }); it('should throw an error', async () => { @@ -511,7 +505,11 @@ describe('SubmissionRepository', () => { const submissionRepository = new SubmissionRepository(mockDBConnection); try { - await submissionRepository.insertSubmissionRecordWithPotentialConflict('aaaa'); + await submissionRepository.insertSubmissionRecordWithPotentialConflict( + '123-456-789', + 'submission name', + 'source system' + ); expect.fail(); } catch (actualError) { expect((actualError as ApiExecuteSQLError).message).to.equal('Failed to get or insert submission record'); diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts index 415d27fe0..58c795170 100644 --- a/api/src/repositories/submission-repository.ts +++ b/api/src/repositories/submission-repository.ts @@ -20,17 +20,11 @@ export interface IDatasetsForReview { keywords: string[]; } -export interface IFeatureSubmission { +export interface ISubmissionFeature { id: string; type: string; properties: object; -} - -export interface IDatasetSubmission { - id: string; - type: string; - properties: object; - features: IFeatureSubmission[]; + features: ISubmissionFeature[]; } export const DatasetMetadata = z.object({ @@ -282,7 +276,7 @@ export const SubmissionMessageRecord = z.object({ submission_id: z.number(), label: z.string(), message: z.string(), - data: z.object({}).nullable(), + data: z.record(z.string(), z.any()).nullable(), create_date: z.string(), create_user: z.number(), update_date: z.string().nullable(), @@ -339,26 +333,31 @@ export class SubmissionRepository extends BaseRepository { } /** - * Insert a new submission record, returning the record having the matching UUID if it already exists. - * - * Because `ON CONFLICT ... DO NOTHING` fails to yield the submission_id, the query simply updates the - * uuid with the given value in the case that they match, which allows us to retrieve the submission_id - * and infer that the query ran successfully. + * Insert a new submission record. * * @param {string} uuid + * @param {string} name + * @param {string} sourceSystem * @return {*} {Promise<{ submission_id: number }>} * @memberof SubmissionRepository */ - async insertSubmissionRecordWithPotentialConflict(uuid: string): Promise<{ submission_id: number }> { + async insertSubmissionRecordWithPotentialConflict( + uuid: string, + name: string, + sourceSystem: string + ): Promise<{ submission_id: number }> { const sqlStatement = SQL` INSERT INTO submission ( uuid, - publish_timestamp + submitted_timestamp, + name, + source_system ) VALUES ( ${uuid}, - now() + now(), + ${name}, + ${sourceSystem} ) - ON CONFLICT (uuid) DO UPDATE SET publish_timestamp = now() RETURNING submission_id; `; @@ -379,7 +378,7 @@ export class SubmissionRepository extends BaseRepository { * Insert a new submission feature record. * * @param {number} submissionId - * @param {IFeatureSubmission} feature + * @param {ISubmissionFeature} feature * @param {number} featureTypeId * @return {*} {Promise<{ submission_feature_id: number }>} * @memberof SubmissionRepository @@ -387,7 +386,7 @@ export class SubmissionRepository extends BaseRepository { async insertSubmissionFeatureRecord( submissionId: number, featureTypeId: number, - feature: IFeatureSubmission + feature: ISubmissionFeature['properties'] ): Promise<{ submission_feature_id: number }> { const sqlStatement = SQL` INSERT INTO submission_feature ( diff --git a/api/src/services/artifact-service.ts b/api/src/services/artifact-service.ts index 782a1ea06..1a1744081 100644 --- a/api/src/services/artifact-service.ts +++ b/api/src/services/artifact-service.ts @@ -84,7 +84,11 @@ export class ArtifactService extends DBService { }); // Create a new submission for the artifact collection - const { submission_id } = await this.submissionService.insertSubmissionRecordWithPotentialConflict(dataPackageId); + const { submission_id } = await this.submissionService.insertSubmissionRecordWithPotentialConflict( + dataPackageId, + 'TODO_Temp', + 'TODO_Temp' + ); // Upload the artifact to S3 await uploadFileToS3(file, s3Key, { filename: file.originalname }); diff --git a/api/src/services/submission-service.test.ts b/api/src/services/submission-service.test.ts index ba0b88e3e..16738d843 100644 --- a/api/src/services/submission-service.test.ts +++ b/api/src/services/submission-service.test.ts @@ -54,7 +54,11 @@ describe('SubmissionService', () => { .stub(SubmissionRepository.prototype, 'insertSubmissionRecordWithPotentialConflict') .resolves({ submission_id: 1 }); - const response = await submissionService.insertSubmissionRecordWithPotentialConflict('aaaa'); + const response = await submissionService.insertSubmissionRecordWithPotentialConflict( + '123-456-789', + 'submission name', + 'source systemF' + ); expect(repo).to.be.calledOnce; expect(response).to.be.eql({ submission_id: 1 }); diff --git a/api/src/services/submission-service.ts b/api/src/services/submission-service.ts index 84315a00f..a49388816 100644 --- a/api/src/services/submission-service.ts +++ b/api/src/services/submission-service.ts @@ -5,9 +5,9 @@ import { IDBConnection } from '../database/db'; import { ApiExecuteSQLError } from '../errors/api-error'; import { IDatasetsForReview, - IFeatureSubmission, IHandlebarsTemplates, ISourceTransformModel, + ISubmissionFeature, ISubmissionJobQueueRecord, ISubmissionMetadataRecord, ISubmissionModel, @@ -61,24 +61,30 @@ export class SubmissionService extends DBService { * in the database. * * @param {string} uuid + * @param {string} name + * @param {string} sourceSystem * @return {*} {Promise<{ submission_id: number }>} * @memberof SubmissionService */ - async insertSubmissionRecordWithPotentialConflict(uuid: string): Promise<{ submission_id: number }> { - return this.submissionRepository.insertSubmissionRecordWithPotentialConflict(uuid); + async insertSubmissionRecordWithPotentialConflict( + uuid: string, + name: string, + sourceSystem: string + ): Promise<{ submission_id: number }> { + return this.submissionRepository.insertSubmissionRecordWithPotentialConflict(uuid, name, sourceSystem); } /** * insert submission feature record * * @param {number} submissionId - * @param {IFeatureSubmission[]} submissionFeature + * @param {ISubmissionFeature[]} submissionFeature * @return {*} {Promise<{ submission_feature_id: number }[]>} * @memberof SubmissionService */ async insertSubmissionFeatureRecords( submissionId: number, - submissionFeature: IFeatureSubmission[] + submissionFeature: ISubmissionFeature[] ): Promise<{ submission_feature_id: number }[]> { const promise = submissionFeature.map(async (feature) => { const featureTypeId = await this.submissionRepository.getFeatureTypeIdByName(feature.type); @@ -86,7 +92,7 @@ export class SubmissionService extends DBService { return this.submissionRepository.insertSubmissionFeatureRecord( submissionId, featureTypeId.feature_type_id, - feature + feature.properties ); }); diff --git a/api/src/services/validation-service.test.ts b/api/src/services/validation-service.test.ts index afaba18b4..b0486607a 100644 --- a/api/src/services/validation-service.test.ts +++ b/api/src/services/validation-service.test.ts @@ -2,8 +2,13 @@ import chai, { expect } from 'chai'; import { describe } from 'mocha'; import sinon from 'sinon'; import sinonChai from 'sinon-chai'; -import { IDatasetSubmission } from '../repositories/submission-repository'; -import { IInsertStyleSchema, IStyleModel, ValidationRepository } from '../repositories/validation-repository'; +import { ISubmissionFeature } from '../repositories/submission-repository'; +import { + IFeatureProperties, + IInsertStyleSchema, + IStyleModel, + ValidationRepository +} from '../repositories/validation-repository'; import { DWCArchive } from '../utils/media/dwc/dwc-archive-file'; import * as validatorParser from '../utils/media/validation/validation-schema-parser'; import { getMockDBConnection } from '../__mocks__/db'; @@ -108,21 +113,19 @@ describe('ValidationService', () => { }); }); - describe('validateDatasetSubmission', () => { + describe('validateSubmissionFeatures', () => { it('should return false if the dataset is invalid', async () => { const mockDBConnection = getMockDBConnection(); - const getFeatureValidationPropertiesSpy = sinon.spy( - ValidationService.prototype, - 'getFeatureValidationProperties' - ); - - const validatePropertiesStub = sinon.stub(ValidationService.prototype, 'validateProperties').returns(true); + const validateSubmissionFeatureStub = sinon + .stub(ValidationService.prototype, 'validateSubmissionFeature') + .throws(new Error('validation error')); const mockDatasetProperties = { name: 'dataset name', start_date: '2023-12-22' }; + const mockObservationProperties1 = { count: 11, sex: 'male', @@ -140,70 +143,66 @@ describe('ValidationService', () => { ] } }; + const mockObservationSubmissionFeature1 = { + id: '1', + type: 'observation', + properties: mockObservationProperties1, + features: [] + }; + const mockObservationProperties2 = { count: 22, sex: 'female', geometry: { - type: 'Feature' - // Invalid geometry + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: {}, + geometry: { + coordinates: [-125.44339737241725, 49.36887682703687], + type: 'Point' + } + } + ] } }; - const mockDataset: IDatasetSubmission = { + const mockObservationSubmissionFeature2 = { + id: '2', + type: 'observation', + properties: mockObservationProperties2, + features: [] + }; + + const mockDatasetSubmissionFeature = { id: '123', type: 'dataset', properties: mockDatasetProperties, - features: [ - { id: '1', type: 'observation', properties: mockObservationProperties1 }, - { id: '2', type: 'observation', properties: mockObservationProperties2 } - ] + features: [mockObservationSubmissionFeature1, mockObservationSubmissionFeature2] }; - const validationService = new ValidationService(mockDBConnection); + const mockSubmissionFeatures: ISubmissionFeature[] = [mockDatasetSubmissionFeature]; - const mockDatasetValidationProperties = [ - { name: 'name', display_name: '', description: '', type: 'string' }, - { name: 'start_date', display_name: '', description: '', type: 'datetime' } - ]; - const mockObservationValidationProperties = [ - { name: 'count', display_name: '', description: '', type: 'number' }, - { name: 'sex', display_name: '', description: '', type: 'string' }, - { name: 'geometry', display_name: '', description: '', type: 'spatial' } - ]; - // Set cache for dataset type - validationService.validationPropertiesCache.set('dataset', mockDatasetValidationProperties); - // Set cache for observation type - validationService.validationPropertiesCache.set('observation', mockObservationValidationProperties); + const validationService = new ValidationService(mockDBConnection); - const response = await validationService.validateDatasetSubmission(mockDataset); + const response = await validationService.validateSubmissionFeatures(mockSubmissionFeatures); - expect(response).to.be.true; - expect(getFeatureValidationPropertiesSpy).to.have.been.calledWith('dataset'); - expect(getFeatureValidationPropertiesSpy).to.have.been.calledWith('observation'); - expect(validatePropertiesStub).to.have.been.calledWith(mockDatasetValidationProperties, mockDatasetProperties); - expect(validatePropertiesStub).to.have.been.calledWith( - mockObservationValidationProperties, - mockObservationProperties1 - ); - expect(validatePropertiesStub).to.have.been.calledWith( - mockObservationValidationProperties, - mockObservationProperties2 - ); + expect(validateSubmissionFeatureStub).to.have.been.calledWith(mockSubmissionFeatures); + expect(response).to.be.false; }); it('should return true if the dataset is valid', async () => { const mockDBConnection = getMockDBConnection(); - const getFeatureValidationPropertiesSpy = sinon.spy( - ValidationService.prototype, - 'getFeatureValidationProperties' - ); - - const validatePropertiesStub = sinon.stub(ValidationService.prototype, 'validateProperties').returns(true); + const validateSubmissionFeatureStub = sinon + .stub(ValidationService.prototype, 'validateSubmissionFeature') + .resolves(true); const mockDatasetProperties = { name: 'dataset name', start_date: '2023-12-22' }; + const mockObservationProperties1 = { count: 11, sex: 'male', @@ -221,6 +220,13 @@ describe('ValidationService', () => { ] } }; + const mockObservationSubmissionFeature1 = { + id: '1', + type: 'observation', + properties: mockObservationProperties1, + features: [] + }; + const mockObservationProperties2 = { count: 22, sex: 'female', @@ -238,46 +244,76 @@ describe('ValidationService', () => { ] } }; - const mockDataset: IDatasetSubmission = { + const mockObservationSubmissionFeature2 = { + id: '2', + type: 'observation', + properties: mockObservationProperties2, + features: [] + }; + + const mockDatasetSubmissionFeature = { id: '123', type: 'dataset', properties: mockDatasetProperties, - features: [ - { id: '1', type: 'observation', properties: mockObservationProperties1 }, - { id: '2', type: 'observation', properties: mockObservationProperties2 } - ] + features: [mockObservationSubmissionFeature1, mockObservationSubmissionFeature2] }; + const mockSubmissionFeatures: ISubmissionFeature[] = [mockDatasetSubmissionFeature]; + const validationService = new ValidationService(mockDBConnection); - const mockDatasetValidationProperties = [ - { name: 'name', display_name: '', description: '', type: 'string' }, - { name: 'start_date', display_name: '', description: '', type: 'datetime' } - ]; - const mockObservationValidationProperties = [ - { name: 'count', display_name: '', description: '', type: 'number' }, - { name: 'sex', display_name: '', description: '', type: 'string' }, - { name: 'geometry', display_name: '', description: '', type: 'spatial' } + const response = await validationService.validateSubmissionFeatures(mockSubmissionFeatures); + + expect(validateSubmissionFeatureStub).to.have.been.calledWith({ ...mockDatasetSubmissionFeature, features: [] }); + expect(validateSubmissionFeatureStub).to.have.been.calledWith({ + ...mockObservationSubmissionFeature1, + features: [] + }); + expect(validateSubmissionFeatureStub).to.have.been.calledWith({ + ...mockObservationSubmissionFeature2, + features: [] + }); + expect(response).to.be.true; + }); + }); + + describe('validateSubmissionFeature', () => { + it('fetches validation properties and calls validate', async () => { + const mockDBConnection = getMockDBConnection(); + + const mockFeatureProperties: IFeatureProperties[] = [ + { + name: 'field1', + display_name: 'Field 1', + description: 'A Field 1', + type: 'string' + } ]; - // Set cache for dataset type - validationService.validationPropertiesCache.set('dataset', mockDatasetValidationProperties); - // Set cache for observation type - validationService.validationPropertiesCache.set('observation', mockObservationValidationProperties); - const response = await validationService.validateDatasetSubmission(mockDataset); + const getFeatureValidationPropertiesStub = sinon + .stub(ValidationService.prototype, 'getFeatureValidationProperties') + .resolves(mockFeatureProperties); - expect(response).to.be.true; - expect(getFeatureValidationPropertiesSpy).to.have.been.calledWith('dataset'); - expect(getFeatureValidationPropertiesSpy).to.have.been.calledWith('observation'); - expect(validatePropertiesStub).to.have.been.calledWith(mockDatasetValidationProperties, mockDatasetProperties); - expect(validatePropertiesStub).to.have.been.calledWith( - mockObservationValidationProperties, - mockObservationProperties1 - ); - expect(validatePropertiesStub).to.have.been.calledWith( - mockObservationValidationProperties, - mockObservationProperties2 - ); + const validatePropertiesStub = sinon.stub(ValidationService.prototype, 'validateProperties').returns(true); + + const mockSubmissionProperties = {}; + const mockSubmissionFeature = { + id: '1', + type: 'feature type', + properties: mockSubmissionProperties, + features: [ + { id: '2', type: 'child feature type', properties: {}, features: [] }, + { id: '3', type: 'child feature type', properties: {}, features: [] } + ] + }; + + const validationService = new ValidationService(mockDBConnection); + + const result = await validationService.validateSubmissionFeature(mockSubmissionFeature); + + expect(result).to.be.true; + expect(getFeatureValidationPropertiesStub).to.have.been.calledOnceWith('feature type'); + expect(validatePropertiesStub).to.have.been.calledOnceWith(mockFeatureProperties, mockSubmissionProperties); }); }); diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 07189a64b..b4f0da340 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -1,5 +1,6 @@ +import { JSONPath } from 'jsonpath-plus'; import { IDBConnection } from '../database/db'; -import { IDatasetSubmission } from '../repositories/submission-repository'; +import { ISubmissionFeature } from '../repositories/submission-repository'; import { IFeatureProperties, IInsertStyleSchema, @@ -25,43 +26,67 @@ export class ValidationService extends DBService { } /** - * Validate dataset submission + * Validate submission features. * - * @param {IDatasetSubmission} dataset + * @param {ISubmissionFeature[]} submissionFeatures * @return {*} {Promise} * @memberof ValidationService */ - async validateDatasetSubmission(dataset: IDatasetSubmission): Promise { - // validate dataset.type is 'dataset' - const datasetFeatureType = dataset.type; - - // get dataset validation properties - const datasetValidationProperties = await this.getFeatureValidationProperties(datasetFeatureType); - - // get features in dataset - const features = dataset.features; + async validateSubmissionFeatures(submissionFeatures: ISubmissionFeature[]): Promise { + // Generate an array of all paths to all elements which contain a 'features' property + const allFeaturesPaths: string[] = JSONPath({ + path: '$..[?(@.features)]', + flatten: true, + resultType: 'path', + json: submissionFeatures + }); + + // TODO Change name of submission 'features' field?? + // Remove paths which actually point to GeoJson features, and not submission features. This step is only necessary + // because the submission 'features' field collides with the GeoJSON 'features' field. Could be solved by picking a + // different name for submission 'features'. + const cleanFeaturePaths = allFeaturesPaths.filter((path) => { + return /\[\d+\]$/.test(path); + }); try { - // validate dataset properties - await this.validateProperties(datasetValidationProperties, dataset.properties); - - // validate features - for (const feature of features) { - const featureType = feature.type; - - // get feature validation properties - const featureValidationProperties = await this.getFeatureValidationProperties(featureType); - - // validate feature properties - await this.validateProperties(featureValidationProperties, feature.properties); + for (const path of cleanFeaturePaths) { + // Fetch a submissionFeature object + const node: ISubmissionFeature[] = JSONPath({ path: path, resultType: 'value', json: submissionFeatures }); + // We expect the 'path' to resolve an array of 1 item + const nodeWithoutFeatures = { ...node[0], features: [] }; + // Validate the submissioNFeature object + await this.validateSubmissionFeature(nodeWithoutFeatures); } - } catch (error) { - console.error(error); + } catch { + // Not all submission features are valid return false; } + + // All submission features are valid return true; } + /** + * Validate a submission feature (not including its child features). + * + * @param {ISubmissionFeature} submissionFeature + * @return {*} {Promise} + * @memberof ValidationService + */ + async validateSubmissionFeature(submissionFeature: ISubmissionFeature): Promise { + const validationProperties = await this.getFeatureValidationProperties(submissionFeature.type); + return this.validateProperties(validationProperties, submissionFeature.properties); + } + + /** + * Validate the properties of a submission feature. + * + * @param {IFeatureProperties[]} properties + * @param {*} dataProperties + * @return {*} {boolean} `true` if the submission feature is valid, `false` otherwise. + * @memberof ValidationService + */ validateProperties(properties: IFeatureProperties[], dataProperties: any): boolean { console.log('dataProperties', dataProperties); console.log('properties', properties); diff --git a/api/src/zod-schema/geoJsonZodSchema.ts b/api/src/zod-schema/geoJsonZodSchema.ts index 44408d817..101f9ba86 100644 --- a/api/src/zod-schema/geoJsonZodSchema.ts +++ b/api/src/zod-schema/geoJsonZodSchema.ts @@ -38,20 +38,20 @@ export const GeoJSONMultiPolygonZodSchema = z.object({ export const GeoJSONGeometryCollectionZodSchema = z.object({ type: z.enum(['GeometryCollection']), - geometries: z.array(z.object({})), + geometries: z.array(z.record(z.string(), z.any())), bbox: z.array(z.number()).min(4).optional() }); export const GeoJSONFeatureZodSchema = z.object({ type: z.enum(['Feature']), id: z.union([z.number(), z.string()]).optional(), - properties: z.object({}), - geometry: z.object({}), + properties: z.record(z.string(), z.any()), + geometry: z.record(z.string(), z.any()), bbox: z.array(z.number()).min(4).optional() }); export const GeoJSONFeatureCollectionZodSchema = z.object({ type: z.enum(['FeatureCollection']), - features: z.array(z.object({})), + features: z.array(z.record(z.string(), z.any())), bbox: z.array(z.number()).min(4).optional() }); From ddec497375efb862330b44abd56b0356b4f4f0c7 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Mon, 18 Dec 2023 12:04:07 -0800 Subject: [PATCH 21/37] Fix unit test --- api/src/services/validation-service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/validation-service.test.ts b/api/src/services/validation-service.test.ts index b0486607a..b3ca7a3b2 100644 --- a/api/src/services/validation-service.test.ts +++ b/api/src/services/validation-service.test.ts @@ -187,7 +187,7 @@ describe('ValidationService', () => { const response = await validationService.validateSubmissionFeatures(mockSubmissionFeatures); - expect(validateSubmissionFeatureStub).to.have.been.calledWith(mockSubmissionFeatures); + expect(validateSubmissionFeatureStub).to.have.been.calledWith({ ...mockDatasetSubmissionFeature, features: [] }); expect(response).to.be.false; }); From d3c417aae4ac430a5d9bce7f9d07fa24fdda4e2a Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Mon, 18 Dec 2023 12:48:17 -0800 Subject: [PATCH 22/37] Fix intake validation --- api/src/paths/submission/intake.ts | 2 +- api/src/services/validation-service.ts | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/api/src/paths/submission/intake.ts b/api/src/paths/submission/intake.ts index 58a28673b..a7ba4ec03 100644 --- a/api/src/paths/submission/intake.ts +++ b/api/src/paths/submission/intake.ts @@ -126,7 +126,7 @@ export function submissionIntake(): RequestHandler { // validate the submission if (!(await validationService.validateSubmissionFeatures(submissionFeatures))) { - throw new HTTP400('Invalid submission submission'); + throw new HTTP400('Invalid submission'); } // insert the submission record diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index b4f0da340..1b71360c5 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -7,6 +7,7 @@ import { IStyleModel, ValidationRepository } from '../repositories/validation-repository'; +import { getLogger } from '../utils/logger'; import { ICsvState } from '../utils/media/csv/csv-file'; import { DWCArchive } from '../utils/media/dwc/dwc-archive-file'; import { IMediaState } from '../utils/media/media-file'; @@ -14,6 +15,8 @@ import { ValidationSchemaParser } from '../utils/media/validation/validation-sch import { GeoJSONFeatureCollectionZodSchema } from '../zod-schema/geoJsonZodSchema'; import { DBService } from './db-service'; +const defaultLog = getLogger('services/validation-service'); + export class ValidationService extends DBService { validationRepository: ValidationRepository; validationPropertiesCache: Map; @@ -53,12 +56,23 @@ export class ValidationService extends DBService { for (const path of cleanFeaturePaths) { // Fetch a submissionFeature object const node: ISubmissionFeature[] = JSONPath({ path: path, resultType: 'value', json: submissionFeatures }); + + if (!node?.length) { + continue; + } + // We expect the 'path' to resolve an array of 1 item const nodeWithoutFeatures = { ...node[0], features: [] }; + // Validate the submissioNFeature object - await this.validateSubmissionFeature(nodeWithoutFeatures); + await this.validateSubmissionFeature(nodeWithoutFeatures).catch((error) => { + defaultLog.error({ label: 'validateSubmissionFeature', message: 'error', error }); + // Submission feature is invalid + return false; + }); } - } catch { + } catch (error) { + defaultLog.error({ label: 'validateSubmissionFeatures', message: 'error', error }); // Not all submission features are valid return false; } @@ -88,8 +102,8 @@ export class ValidationService extends DBService { * @memberof ValidationService */ validateProperties(properties: IFeatureProperties[], dataProperties: any): boolean { - console.log('dataProperties', dataProperties); - console.log('properties', properties); + defaultLog.debug({ label: 'validateProperties', message: 'params', properties, dataProperties }); + const throwPropertyError = (property: IFeatureProperties) => { throw new Error(`Property ${property.name} is not of type ${property.type}`); }; From 0cc8cd20a5a58c8d88c55f4089980ed42eceef16 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Mon, 18 Dec 2023 14:13:48 -0800 Subject: [PATCH 23/37] SIMSBIOHUB-379: Addressed PR feedback --- api/src/services/search-index-service.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index d8d1318ca..5e6ede1f1 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -53,7 +53,7 @@ export class SearchIndexService extends DBService { features.forEach((feature) => { const { submission_feature_id } = feature; - Object.entries(feature.data.properties).forEach(([feature_property_name, value]) => { + Object.entries(feature.data).forEach(([feature_property_name, value]) => { const featureProperty = featurePropertyTypeMap[feature_property_name]; if (!featureProperty) { return; @@ -81,17 +81,21 @@ export class SearchIndexService extends DBService { }); }); + const promises = []; + if (datetimeRecords.length) { - this.searchIndexRepository.insertSearchableDatetimeRecords(datetimeRecords); + promises.push(this.searchIndexRepository.insertSearchableDatetimeRecords(datetimeRecords)); } if (numberRecords.length) { - this.searchIndexRepository.insertSearchableNumberRecords(numberRecords); + promises.push(this.searchIndexRepository.insertSearchableNumberRecords(numberRecords)); } if (spatialRecords.length) { - this.searchIndexRepository.insertSearchableSpatialRecords(spatialRecords); + promises.push(this.searchIndexRepository.insertSearchableSpatialRecords(spatialRecords)); } if (stringRecords.length) { - this.searchIndexRepository.insertSearchableStringRecords(stringRecords); + promises.push(this.searchIndexRepository.insertSearchableStringRecords(stringRecords)); } + + await Promise.all(promises); } } From 2bdd82d804c59f119491dac9ac775cf4b4b35d28 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Mon, 18 Dec 2023 14:24:26 -0800 Subject: [PATCH 24/37] SIMSBIOHUB-379: Fix zod schema checking for index table insertion --- api/src/repositories/search-index-respository.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index c41fa34d4..0bf77cc69 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -101,7 +101,7 @@ export class SearchIndexRepository extends BaseRepository { const queryBuilder = getKnex().queryBuilder().insert(datetimeRecords).into('search_datetime').returning('*'); - const response = await this.connection.knex(queryBuilder); + const response = await this.connection.knex(queryBuilder, DatetimeSearchableRecord); if (response.rowCount !== datetimeRecords.length) { throw new ApiExecuteSQLError('Failed to insert searchable datetime records', [ @@ -127,7 +127,7 @@ export class SearchIndexRepository extends BaseRepository { const queryBuilder = getKnex().queryBuilder().insert(numberRecords).into('search_number').returning('*'); - const response = await this.connection.knex(queryBuilder); + const response = await this.connection.knex(queryBuilder, NumberSearchableRecord); if (response.rowCount !== numberRecords.length) { throw new ApiExecuteSQLError('Failed to insert searchable number records', [ @@ -153,7 +153,7 @@ export class SearchIndexRepository extends BaseRepository { const queryBuilder = getKnex().queryBuilder().insert(spatialRecords).into('search_spatial').returning('*'); - const response = await this.connection.knex(queryBuilder); + const response = await this.connection.knex(queryBuilder, SpatialSearchableRecord); if (response.rowCount !== spatialRecords.length) { throw new ApiExecuteSQLError('Failed to insert searchable spatial records', [ @@ -179,7 +179,7 @@ export class SearchIndexRepository extends BaseRepository { const queryBuilder = getKnex().queryBuilder().insert(stringRecords).into('search_string').returning('*'); - const response = await this.connection.knex(queryBuilder); + const response = await this.connection.knex(queryBuilder, StringSearchableRecord); if (response.rowCount !== stringRecords.length) { throw new ApiExecuteSQLError('Failed to insert searchable string records', [ From ed00e7b005392504a1f6b06e016e298b1d72301a Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Mon, 18 Dec 2023 15:05:19 -0800 Subject: [PATCH 25/37] SIMSBIOHUB-379: Promise typing --- api/src/services/search-index-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 5e6ede1f1..2d53ccd0d 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -81,7 +81,7 @@ export class SearchIndexService extends DBService { }); }); - const promises = []; + const promises: Promise[] = []; if (datetimeRecords.length) { promises.push(this.searchIndexRepository.insertSearchableDatetimeRecords(datetimeRecords)); From 6e90031a059f42195fec0712cbb07de792da6cce Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 10:34:31 -0800 Subject: [PATCH 26/37] SIMSBIOHUB-379: Fixed geometry typing --- .../repositories/search-index-respository.ts | 107 +++++++++++++----- api/src/services/search-index-service.ts | 6 +- 2 files changed, 83 insertions(+), 30 deletions(-) diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index 0bf77cc69..22cd7f7da 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -4,6 +4,9 @@ import { getKnex } from '../database/db'; import { ApiExecuteSQLError } from '../errors/api-error'; import { getLogger } from '../utils/logger'; import { BaseRepository } from './base-repository'; +import { generateGeometryCollectionSQL } from '../utils/spatial-utils'; +import { GeoJSONFeatureCollectionZodSchema } from '../zod-schema/geoJsonZodSchema'; +import { Feature } from 'geojson'; const defaultLog = getLogger('repositories/search-index-repository'); @@ -31,57 +34,80 @@ const FeaturePropertyRecordWithPropertyTypeName = FeaturePropertyRecord.extend({ export type FeaturePropertyRecordWithPropertyTypeName = z.infer; -// TODO replace with pre-existing Zod types for geojson -const Geometry = z.object({ - type: z.literal('Point'), - coordinates: z.tuple([z.number(), z.number()]) -}); - -export type Geometry = z.infer; - +/** + * Represents a record in one of the search tables. + */ const SearchableRecord = z.object({ submission_feature_id: z.number(), feature_property_id: z.number(), value: z.unknown(), - create_date: z.date(), + create_date: z.string(), create_user: z.number(), - update_date: z.date().nullable(), - update_user: z.date().nullable(), + update_date: z.string().nullable(), + update_user: z.string().nullable(), revision_count: z.number() }); -type InsertSearchableRecordKey = 'submission_feature_id' | 'value' | 'feature_property_id'; +export type SearchableRecord = z.infer; +const InsertSearchableRecordKeys = { + 'submission_feature_id': true, + 'feature_property_id': true, + 'value': true +} as const; + +/** + * Represents a record in the datetime search table. + */ export const DatetimeSearchableRecord = SearchableRecord.extend({ - search_datetime_id: z.date(), - value: z.date() + search_datetime_id: z.number(), + value: z.string() }); +/** + * Represents a record in the number search table. + */ export const NumberSearchableRecord = SearchableRecord.extend({ search_number_id: z.number(), value: z.number() }); -export const SpatialSearchableRecord = SearchableRecord.extend({ - search_spatial_id: z.number(), - value: Geometry -}); - +/** + * Represents a record in the string search table. + */ export const StringSearchableRecord = SearchableRecord.extend({ search_string_id: z.number(), value: z.string() }); -export type SearchableRecord = z.infer; +/** + * Represents a record in the spatial search table. + * + * Because values from a type `geometry` column are not useful, we elect to never + * return them (`z.never()`). + */ +export const SpatialSearchableRecord = SearchableRecord.extend({ + search_spatial_id: z.number(), + value: z.never() // Geometry represented as a string +}); + +export const InsertDatetimeSearchableRecord = DatetimeSearchableRecord.pick(InsertSearchableRecordKeys); +export const InsertNumberSearchableRecord = NumberSearchableRecord.pick(InsertSearchableRecordKeys); +export const InsertStringSearchableRecord = StringSearchableRecord.pick(InsertSearchableRecordKeys); +export const InsertSpatialSearchableRecord = SpatialSearchableRecord.pick(InsertSearchableRecordKeys) + .extend({ + value: GeoJSONFeatureCollectionZodSchema + }); + export type DatetimeSearchableRecord = z.infer; export type NumberSearchableRecord = z.infer; -export type SpatialSearchableRecord = z.infer; export type StringSearchableRecord = z.infer; +export type SpatialSearchableRecord = z.infer; -export type InsertDatetimeSearchableRecord = Pick; -export type InsertNumberSearchableRecord = Pick; -export type InsertSpatialSearchableRecord = Pick; -export type InsertStringSearchableRecord = Pick; +export type InsertDatetimeSearchableRecord = z.infer; +export type InsertNumberSearchableRecord = z.infer; +export type InsertStringSearchableRecord = z.infer; +export type InsertSpatialSearchableRecord = z.infer; /** * A class for creating searchable records @@ -151,9 +177,36 @@ export class SearchIndexRepository extends BaseRepository { ): Promise { defaultLog.debug({ label: 'insertSearchableSpatialRecords' }); - const queryBuilder = getKnex().queryBuilder().insert(spatialRecords).into('search_spatial').returning('*'); + const query = SQL` + INSERT INTO + search_spatial + ( + submission_feature_id, + feature_property_id, + value + ) + VALUES + `; - const response = await this.connection.knex(queryBuilder, SpatialSearchableRecord); + spatialRecords.forEach((spatialRecord, index) => { + const { + submission_feature_id, + feature_property_id, + value + } = spatialRecord; + + query.append(SQL`( + ${submission_feature_id}, + ${feature_property_id},`); + query.append(generateGeometryCollectionSQL(value.features as Feature[])); + query.append(SQL`)`); + + if (index < spatialRecords.length - 1) { + query.append(SQL`,`); + } + }); + + const response = await this.connection.sql(query, SpatialSearchableRecord); if (response.rowCount !== spatialRecords.length) { throw new ApiExecuteSQLError('Failed to insert searchable spatial records', [ diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 2d53ccd0d..1a556ca7d 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -1,7 +1,7 @@ +import { FeatureCollection } from 'geojson'; import { IDBConnection } from '../database/db'; import { FeaturePropertyRecordWithPropertyTypeName, - Geometry, InsertDatetimeSearchableRecord, InsertNumberSearchableRecord, InsertSpatialSearchableRecord, @@ -63,7 +63,7 @@ export class SearchIndexService extends DBService { switch (feature_property_type_name) { case 'datetime': - datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as Date }); + datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as string }); break; case 'number': @@ -71,7 +71,7 @@ export class SearchIndexService extends DBService { break; case 'spatial': - spatialRecords.push({ submission_feature_id, feature_property_id, value: value as Geometry }); + spatialRecords.push({ submission_feature_id, feature_property_id, value: value as FeatureCollection }); break; case 'string': From c52a14d7b0e87f5c6c4bd7655e68ab3c807eb213 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 19 Dec 2023 10:59:51 -0800 Subject: [PATCH 27/37] Improve JSONPath string in submission validation-service.ts --- api/src/services/validation-service.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/api/src/services/validation-service.ts b/api/src/services/validation-service.ts index 1b71360c5..7d85649b0 100644 --- a/api/src/services/validation-service.ts +++ b/api/src/services/validation-service.ts @@ -36,24 +36,16 @@ export class ValidationService extends DBService { * @memberof ValidationService */ async validateSubmissionFeatures(submissionFeatures: ISubmissionFeature[]): Promise { - // Generate an array of all paths to all elements which contain a 'features' property + // Generate paths to all non-null nodes which contain a 'features' property, ignoring the 'properties' field const allFeaturesPaths: string[] = JSONPath({ - path: '$..[?(@.features)]', + path: "$..[?(@ && @parentProperty != 'properties' && @.features)]", flatten: true, resultType: 'path', json: submissionFeatures }); - // TODO Change name of submission 'features' field?? - // Remove paths which actually point to GeoJson features, and not submission features. This step is only necessary - // because the submission 'features' field collides with the GeoJSON 'features' field. Could be solved by picking a - // different name for submission 'features'. - const cleanFeaturePaths = allFeaturesPaths.filter((path) => { - return /\[\d+\]$/.test(path); - }); - try { - for (const path of cleanFeaturePaths) { + for (const path of allFeaturesPaths) { // Fetch a submissionFeature object const node: ISubmissionFeature[] = JSONPath({ path: path, resultType: 'value', json: submissionFeatures }); From 0e21f946a799e54b2916a1f9e6d64b30dcc78c3b Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 12:30:30 -0800 Subject: [PATCH 28/37] SISMBIOHUB-379: Filtered out null end_date --- api/src/services/search-index-service.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/src/services/search-index-service.ts b/api/src/services/search-index-service.ts index 1a556ca7d..3e4e02a4c 100644 --- a/api/src/services/search-index-service.ts +++ b/api/src/services/search-index-service.ts @@ -63,6 +63,11 @@ export class SearchIndexService extends DBService { switch (feature_property_type_name) { case 'datetime': + if (!value) { + // Datetime value is null or undefined, since the submission system accepts null dates (e.g. `{ end_date: null }`) + return; + } + datetimeRecords.push({ submission_feature_id, feature_property_id, value: value as string }); break; From bdf31b5d7dd818822d327d9ee50bb32a6169b721 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 14:30:44 -0800 Subject: [PATCH 29/37] SIMSBIOHUB-379: Remove console log --- app/src/hooks/api/useDatasetApi.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/hooks/api/useDatasetApi.ts b/app/src/hooks/api/useDatasetApi.ts index bdc2ade0d..3e574b6ab 100644 --- a/app/src/hooks/api/useDatasetApi.ts +++ b/app/src/hooks/api/useDatasetApi.ts @@ -40,7 +40,6 @@ const useDatasetApi = (axios: AxiosInstance) => { */ const getDataset = async (datasetUUID: string): Promise => { const { data } = await axios.get(`api/dataset/${datasetUUID}`); - console.log('data', data); return data; }; From 05336067599a6812aa792942ad7e443e13ed9399 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 16:07:42 -0800 Subject: [PATCH 30/37] SIMSBIOHUB-379: Lint fixes --- .../repositories/search-index-respository.ts | 27 ++++++++----------- app/src/components/layout/header/Header.tsx | 5 ++-- .../components/layout/header/UserControls.tsx | 6 ++--- app/src/utils/Utils.test.ts | 2 +- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/api/src/repositories/search-index-respository.ts b/api/src/repositories/search-index-respository.ts index 22cd7f7da..e4eed3a6e 100644 --- a/api/src/repositories/search-index-respository.ts +++ b/api/src/repositories/search-index-respository.ts @@ -1,12 +1,12 @@ +import { Feature } from 'geojson'; import SQL from 'sql-template-strings'; import { z } from 'zod'; import { getKnex } from '../database/db'; import { ApiExecuteSQLError } from '../errors/api-error'; import { getLogger } from '../utils/logger'; -import { BaseRepository } from './base-repository'; import { generateGeometryCollectionSQL } from '../utils/spatial-utils'; import { GeoJSONFeatureCollectionZodSchema } from '../zod-schema/geoJsonZodSchema'; -import { Feature } from 'geojson'; +import { BaseRepository } from './base-repository'; const defaultLog = getLogger('repositories/search-index-repository'); @@ -51,9 +51,9 @@ const SearchableRecord = z.object({ export type SearchableRecord = z.infer; const InsertSearchableRecordKeys = { - 'submission_feature_id': true, - 'feature_property_id': true, - 'value': true + submission_feature_id: true, + feature_property_id: true, + value: true } as const; /** @@ -82,7 +82,7 @@ export const StringSearchableRecord = SearchableRecord.extend({ /** * Represents a record in the spatial search table. - * + * * Because values from a type `geometry` column are not useful, we elect to never * return them (`z.never()`). */ @@ -94,10 +94,9 @@ export const SpatialSearchableRecord = SearchableRecord.extend({ export const InsertDatetimeSearchableRecord = DatetimeSearchableRecord.pick(InsertSearchableRecordKeys); export const InsertNumberSearchableRecord = NumberSearchableRecord.pick(InsertSearchableRecordKeys); export const InsertStringSearchableRecord = StringSearchableRecord.pick(InsertSearchableRecordKeys); -export const InsertSpatialSearchableRecord = SpatialSearchableRecord.pick(InsertSearchableRecordKeys) - .extend({ - value: GeoJSONFeatureCollectionZodSchema - }); +export const InsertSpatialSearchableRecord = SpatialSearchableRecord.pick(InsertSearchableRecordKeys).extend({ + value: GeoJSONFeatureCollectionZodSchema +}); export type DatetimeSearchableRecord = z.infer; export type NumberSearchableRecord = z.infer; @@ -189,18 +188,14 @@ export class SearchIndexRepository extends BaseRepository { `; spatialRecords.forEach((spatialRecord, index) => { - const { - submission_feature_id, - feature_property_id, - value - } = spatialRecord; + const { submission_feature_id, feature_property_id, value } = spatialRecord; query.append(SQL`( ${submission_feature_id}, ${feature_property_id},`); query.append(generateGeometryCollectionSQL(value.features as Feature[])); query.append(SQL`)`); - + if (index < spatialRecords.length - 1) { query.append(SQL`,`); } diff --git a/app/src/components/layout/header/Header.tsx b/app/src/components/layout/header/Header.tsx index 6e4bb2518..9c0069ba6 100644 --- a/app/src/components/layout/header/Header.tsx +++ b/app/src/components/layout/header/Header.tsx @@ -7,16 +7,16 @@ import Dialog from '@mui/material/Dialog'; import DialogActions from '@mui/material/DialogActions'; import DialogContent from '@mui/material/DialogContent'; import DialogTitle from '@mui/material/DialogTitle'; -import Toolbar from '@mui/material/Toolbar'; import Menu from '@mui/material/Menu'; import MenuItem from '@mui/material/MenuItem'; +import Toolbar from '@mui/material/Toolbar'; import Typography from '@mui/material/Typography'; import headerImageLarge from 'assets/images/gov-bc-logo-horiz.png'; import headerImageSmall from 'assets/images/gov-bc-logo-vert.png'; import { AuthGuard, SystemRoleGuard, UnAuthGuard } from 'components/security/Guards'; import { SYSTEM_ROLE } from 'constants/roles'; -import { Link as RouterLink } from 'react-router-dom'; import { useState } from 'react'; +import { Link as RouterLink } from 'react-router-dom'; import { LoggedInUser, PublicViewUser } from './UserControls'; const Header: React.FC = () => { @@ -90,7 +90,6 @@ const Header: React.FC = () => { ); }; - return ( <> { @@ -76,7 +76,7 @@ export const LoggedInUser = () => { // Unauthenticated public view export const PublicViewUser = () => { - const { keycloakWrapper } = useAuthStateContext() + const { keycloakWrapper } = useAuthStateContext(); const loginUrl = useMemo(() => keycloakWrapper?.getLoginUrl(), [keycloakWrapper]); return ( diff --git a/app/src/utils/Utils.test.ts b/app/src/utils/Utils.test.ts index 67c12bf72..12677df7e 100644 --- a/app/src/utils/Utils.test.ts +++ b/app/src/utils/Utils.test.ts @@ -1,5 +1,6 @@ import { DATE_FORMAT } from 'constants/dateTimeFormats'; import { IConfig } from 'contexts/configContext'; +import { SYSTEM_IDENTITY_SOURCE } from 'hooks/useKeycloakWrapper'; import { LatLngBounds, LatLngLiteral } from 'leaflet'; import { downloadFile, @@ -18,7 +19,6 @@ import { safeJSONParse, safeJSONStringify } from './Utils'; -import { SYSTEM_IDENTITY_SOURCE } from 'hooks/useKeycloakWrapper'; describe('ensureProtocol', () => { it('upgrades the URL if string begins with `http://`', async () => { From 8ea17216c0e6224a374a439184be25e4877f000e Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 16:09:46 -0800 Subject: [PATCH 31/37] SIMSBIOHUB-379: Added test check --- api/src/services/search-index-service.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts index 0885e8844..a89263e3f 100644 --- a/api/src/services/search-index-service.test.ts +++ b/api/src/services/search-index-service.test.ts @@ -35,6 +35,7 @@ describe('SearchIndexService', () => { description: 'Desc1', taxonomy: 1001, start_date: new Date('2000-01-01'), + end_date: new Date('2020-01-02'), geometry: { type: 'Point', coordinates: [11, 11] }, count: 60, latitude: 11, @@ -65,6 +66,7 @@ describe('SearchIndexService', () => { description: 'Desc2', taxonomy: 1002, start_date: new Date('2001-01-01'), + end_date: null, geometry: { type: 'Point', coordinates: [22, 22] }, count: 70, latitude: 22, @@ -310,6 +312,11 @@ describe('SearchIndexService', () => { feature_property_id: 5, // Start Date value: new Date('2000-01-01') }, + { + submission_feature_id: 11111, + feature_property_id: 6, // End Date + value: new Date('2000-01-02') + }, { submission_feature_id: 22222, feature_property_id: 5, // Start Date From 987c7e9806e8b9cf338d14621390074cf99491be Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 16:25:19 -0800 Subject: [PATCH 32/37] SIMSBIOHUB-379: Fix compilation errors in tests --- api/src/paths/submission/intake.test.ts | 12 ++++++------ api/src/services/submission-service.test.ts | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/api/src/paths/submission/intake.test.ts b/api/src/paths/submission/intake.test.ts index e39480684..b70476100 100644 --- a/api/src/paths/submission/intake.test.ts +++ b/api/src/paths/submission/intake.test.ts @@ -51,8 +51,8 @@ describe('intake', () => { sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); sinon.stub(keycloakUtils, 'getKeycloakSource').resolves(true); - const validateDatasetSubmissionStub = sinon - .stub(ValidationService.prototype, 'validateDatasetSubmission') + const validateSubmissionFeaturesStub = sinon + .stub(ValidationService.prototype, 'validateSubmissionFeatures') .resolves(false); const requestHandler = intake.submissionIntake(); @@ -71,7 +71,7 @@ describe('intake', () => { expect.fail(); } catch (error) { - expect(validateDatasetSubmissionStub).to.have.been.calledOnce; + expect(validateSubmissionFeaturesStub).to.have.been.calledOnce; expect((error as HTTPError).status).to.equal(400); expect((error as HTTPError).message).to.equal('Invalid submission submission'); } @@ -83,8 +83,8 @@ describe('intake', () => { sinon.stub(db, 'getDBConnection').returns(dbConnectionObj); sinon.stub(keycloakUtils, 'getKeycloakSource').resolves(true); - const validateDatasetSubmissionStub = sinon - .stub(ValidationService.prototype, 'validateDatasetSubmission') + const validateSubmissionFeaturesStub = sinon + .stub(ValidationService.prototype, 'validateSubmissionFeatures') .resolves(true); const insertSubmissionRecordWithPotentialConflictStub = sinon @@ -108,7 +108,7 @@ describe('intake', () => { await requestHandler(mockReq, mockRes, mockNext); - expect(validateDatasetSubmissionStub).to.have.been.calledOnce; + expect(validateSubmissionFeaturesStub).to.have.been.calledOnce; expect(insertSubmissionRecordWithPotentialConflictStub).to.have.been.calledOnce; expect(insertSubmissionFeatureRecordsStub).to.have.been.calledOnce; expect(mockRes.statusValue).to.eql(200); diff --git a/api/src/services/submission-service.test.ts b/api/src/services/submission-service.test.ts index 527d7da82..788788583 100644 --- a/api/src/services/submission-service.test.ts +++ b/api/src/services/submission-service.test.ts @@ -83,7 +83,8 @@ describe('SubmissionService', () => { { id: '1', type: 'string', - properties: {} + properties: {}, + features: [] } ]); From bbedbad5ab3e357c3c25e01cc6093bad3d676b77 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 16:35:54 -0800 Subject: [PATCH 33/37] SIMSBIOHUB-379: Fixed test --- api/src/services/search-index-service.test.ts | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts index a89263e3f..a76c34844 100644 --- a/api/src/services/search-index-service.test.ts +++ b/api/src/services/search-index-service.test.ts @@ -9,7 +9,7 @@ import { SearchIndexService } from './search-index-service'; chai.use(sinonChai); -describe('SearchIndexService', () => { +describe.only('SearchIndexService', () => { afterEach(() => { sinon.restore(); }); @@ -28,19 +28,15 @@ describe('SearchIndexService', () => { submission_id: 1, // Mock submission feature_type_id: 1, // dataset, observation, etc. data: { - id: 100, - type: 'some_random_thing', - properties: { - name: 'Ardvark', - description: 'Desc1', - taxonomy: 1001, - start_date: new Date('2000-01-01'), - end_date: new Date('2020-01-02'), - geometry: { type: 'Point', coordinates: [11, 11] }, - count: 60, - latitude: 11, - longitude: 11 - } + name: 'Ardvark', + description: 'Desc1', + taxonomy: 1001, + start_date: new Date('2000-01-01'), + end_date: new Date('2000-01-02'), + geometry: { type: 'Point', coordinates: [11, 11] }, + count: 60, + latitude: 11, + longitude: 11 }, parent_submission_feature_id: null, record_effective_date: '', @@ -59,19 +55,15 @@ describe('SearchIndexService', () => { submission_id: 1, // Mock submission feature_type_id: 1, // dataset, observation, etc. data: { - id: 200, - type: 'another_random_thing', - properties: { - name: 'Buffalo', - description: 'Desc2', - taxonomy: 1002, - start_date: new Date('2001-01-01'), - end_date: null, - geometry: { type: 'Point', coordinates: [22, 22] }, - count: 70, - latitude: 22, - longitude: 22 - } + name: 'Buffalo', + description: 'Desc2', + taxonomy: 1002, + start_date: new Date('2001-01-01'), + end_date: null, + geometry: { type: 'Point', coordinates: [22, 22] }, + count: 70, + latitude: 22, + longitude: 22 }, parent_submission_feature_id: null, record_effective_date: '', From 58261520db56c97aae6627eacc51df5c82b2fc10 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 19 Dec 2023 16:39:23 -0800 Subject: [PATCH 34/37] Remove .only --- api/src/services/search-index-service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/search-index-service.test.ts b/api/src/services/search-index-service.test.ts index a76c34844..7287209c7 100644 --- a/api/src/services/search-index-service.test.ts +++ b/api/src/services/search-index-service.test.ts @@ -9,7 +9,7 @@ import { SearchIndexService } from './search-index-service'; chai.use(sinonChai); -describe.only('SearchIndexService', () => { +describe('SearchIndexService', () => { afterEach(() => { sinon.restore(); }); From d5fca7922df9fc8d3da85f35707cc6fc3ffd5521 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 16:43:04 -0800 Subject: [PATCH 35/37] SIMSBIOHUB-379: Fixed typo in test --- api/src/paths/submission/intake.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/paths/submission/intake.test.ts b/api/src/paths/submission/intake.test.ts index b70476100..44d3a8e85 100644 --- a/api/src/paths/submission/intake.test.ts +++ b/api/src/paths/submission/intake.test.ts @@ -73,7 +73,7 @@ describe('intake', () => { } catch (error) { expect(validateSubmissionFeaturesStub).to.have.been.calledOnce; expect((error as HTTPError).status).to.equal(400); - expect((error as HTTPError).message).to.equal('Invalid submission submission'); + expect((error as HTTPError).message).to.equal('Invalid submission'); } }); From 4507700b675d286fe32af59afc1015ea8dfd128d Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Tue, 19 Dec 2023 16:52:09 -0800 Subject: [PATCH 36/37] SIMSBIOHUB-379: Added missing stub for intake endpoint test --- api/src/paths/submission/intake.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/src/paths/submission/intake.test.ts b/api/src/paths/submission/intake.test.ts index 44d3a8e85..5046511e2 100644 --- a/api/src/paths/submission/intake.test.ts +++ b/api/src/paths/submission/intake.test.ts @@ -9,6 +9,7 @@ import { ValidationService } from '../../services/validation-service'; import * as keycloakUtils from '../../utils/keycloak-utils'; import { getMockDBConnection, getRequestHandlerMocks } from '../../__mocks__/db'; import * as intake from './intake'; +import { SearchIndexService } from '../../services/search-index-service'; chai.use(sinonChai); @@ -95,6 +96,10 @@ describe('intake', () => { .stub(SubmissionService.prototype, 'insertSubmissionFeatureRecords') .resolves(); + const indexFeaturesBySubmissionIdStub = sinon + .stub(SearchIndexService.prototype, 'indexFeaturesBySubmissionId') + .resolves(); + const requestHandler = intake.submissionIntake(); const { mockReq, mockRes, mockNext } = getRequestHandlerMocks(); @@ -111,6 +116,7 @@ describe('intake', () => { expect(validateSubmissionFeaturesStub).to.have.been.calledOnce; expect(insertSubmissionRecordWithPotentialConflictStub).to.have.been.calledOnce; expect(insertSubmissionFeatureRecordsStub).to.have.been.calledOnce; + expect(indexFeaturesBySubmissionIdStub).to.have.been.calledOnce; expect(mockRes.statusValue).to.eql(200); expect(mockRes.jsonValue).to.eql({ submission_id: 1 }); }); From 03f670597982dccb14af437157624ac513a92f72 Mon Sep 17 00:00:00 2001 From: Curtis Upshall Date: Wed, 20 Dec 2023 07:43:55 -0800 Subject: [PATCH 37/37] SIMSBIOHUB-379: Ran linter --- api/src/paths/submission/intake.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/paths/submission/intake.test.ts b/api/src/paths/submission/intake.test.ts index 5046511e2..4c673ae48 100644 --- a/api/src/paths/submission/intake.test.ts +++ b/api/src/paths/submission/intake.test.ts @@ -4,12 +4,12 @@ import sinon from 'sinon'; import sinonChai from 'sinon-chai'; import * as db from '../../database/db'; import { HTTPError } from '../../errors/http-error'; +import { SearchIndexService } from '../../services/search-index-service'; import { SubmissionService } from '../../services/submission-service'; import { ValidationService } from '../../services/validation-service'; import * as keycloakUtils from '../../utils/keycloak-utils'; import { getMockDBConnection, getRequestHandlerMocks } from '../../__mocks__/db'; import * as intake from './intake'; -import { SearchIndexService } from '../../services/search-index-service'; chai.use(sinonChai);