SIMSBIOHUB-424: Support Artifact Intake + Misc Enhancements (#230)

Support artifact submission intake. Make uuid a source submitted field for submission. make source_id optional for submission_features. Add required_value column to feature_type_property to mark feature properties as required vs optional. Update validation logic to account for required vs optional properties. Fix submission ingest not assigning parent submission feature id. Adjust get unreviewed/completed/published submission SQL. --------- Co-authored-by: Curtis Upshall <[email protected]> Co-authored-by: Alfred Rosenthal <[email protected]>
bcgov · Jan 23, 2024 · b42969b · b42969b
1 parent 9ef67c6
commit b42969b
Show file tree

Hide file tree

Showing 76 changed files with 3,684 additions and 2,342 deletions.
diff --git a/api/package.json b/api/package.json
@@ -44,7 +44,7 @@
     "fast-xml-parser": "~4.1.3",
     "fastq": "^1.15.0",
     "json-schema-traverse": "^1.0.0",
-    "jsonpath-plus": "~7.2.0",
+    "jsonpath-plus": "^8.0.0",
     "jsonwebtoken": "~8.5.1",
     "jwks-rsa": "~2.0.5",
     "knex": "~2.4.2",

diff --git a/api/src/app.ts b/api/src/app.ts
@@ -5,9 +5,10 @@ import { OpenAPIV3 } from 'openapi-types';
 import swaggerUIExperss from 'swagger-ui-express';
 import { defaultPoolConfig, initDBPool } from './database/db';
 import { initDBConstants } from './database/db-constants';
-import { ensureHTTPError, HTTPErrorType } from './errors/http-error';
+import { ensureHTTPError, HTTP400, HTTPErrorType } from './errors/http-error';
 import { rootAPIDoc } from './openapi/root-api-doc';
 import { authenticateRequest, authenticateRequestOptional } from './request-handlers/security/authentication';
+import { scanFileForVirus } from './utils/file-utils';
 import { getLogger } from './utils/logger';
 
 const defaultLog = getLogger('app');
@@ -53,22 +54,29 @@ const openAPIFramework = initialize({
   docsPath: '/raw-api-docs', // path to view raw openapi spec
   consumesMiddleware: {
     'application/json': express.json({ limit: MAX_REQ_BODY_SIZE }),
-    'multipart/form-data': function (req, res, next) {
+    'multipart/form-data': async function (req, res, next) {
       const multerRequestHandler = multer({
-        storage: multer.memoryStorage(),
+        storage: multer.memoryStorage(), // TOOD change to local/PVC storage and stream file uploads to S3?
         limits: { fileSize: MAX_UPLOAD_FILE_SIZE }
       }).array('media', MAX_UPLOAD_NUM_FILES);
 
-      multerRequestHandler(req, res, (error?: any) => {
+      return multerRequestHandler(req, res, async function (error?: any) {
         if (error) {
           return next(error);
         }
 
-        if (req.files && req.files.length) {
+        const promises = (req.files as Express.Multer.File[]).map(async function (file) {
           // Set original request file field to empty string to satisfy OpenAPI validation
           // See: https://www.npmjs.com/package/express-openapi#argsconsumesmiddleware
-          (req.files as Express.Multer.File[]).forEach((file) => (req.body[file.fieldname] = ''));
-        }
+          req.body[file.fieldname] = '';
+
+          // Scan file for malicious content, if enabled
+          if (!(await scanFileForVirus(file))) {
+            throw new HTTP400('Malicious file content detected.', [{ file_name: file.originalname }]);
+          }
+        });
+
+        await Promise.all(promises);
 
         return next();
       });

diff --git a/api/src/database/db-utils.ts b/api/src/database/db-utils.ts
@@ -41,7 +41,7 @@ export const syncErrorWrapper =
  */
 const parseError = (error: any) => {
   if (error instanceof z.ZodError) {
-    throw new ApiExecuteSQLError('SQL response failed schema check', [error]);
+    throw new ApiExecuteSQLError('SQL response failed schema check', [error as Record<string, any>]);
   }
 
   if (error.message === 'CONCURRENCY_EXCEPTION') {

diff --git a/api/src/database/db.ts b/api/src/database/db.ts
@@ -392,7 +392,7 @@ export const getDBConnection = function (keycloakToken: object): IDBConnection {
 
       _systemUserId = response?.rows?.[0].api_set_context;
     } catch (error) {
-      throw new ApiExecuteSQLError('Failed to set user context', [error as object]);
+      throw new ApiExecuteSQLError('Failed to set user context', [error as Record<string, unknown>]);
     }
   };
 

diff --git a/api/src/errors/api-error.ts b/api/src/errors/api-error.ts
@@ -6,9 +6,9 @@ export enum ApiErrorType {
 }
 
 export class ApiError extends Error {
-  errors?: (string | object)[];
+  errors?: (string | Record<string, unknown>)[];
 
-  constructor(name: ApiErrorType, message: string, errors?: (string | object)[], stack?: string) {
+  constructor(name: ApiErrorType, message: string, errors?: (string | Record<string, unknown>)[], stack?: string) {
     super(message);
 
     this.name = name;
@@ -33,7 +33,7 @@ export class ApiError extends Error {
  * @extends {ApiError}
  */
 export class ApiGeneralError extends ApiError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(ApiErrorType.GENERAL, message, errors);
   }
 }
@@ -46,7 +46,7 @@ export class ApiGeneralError extends ApiError {
  * @extends {ApiError}
  */
 export class ApiUnknownError extends ApiError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(ApiErrorType.UNKNOWN, message, errors);
   }
 }
@@ -63,7 +63,7 @@ export class ApiUnknownError extends ApiError {
  * @extends {ApiError}
  */
 export class ApiExecuteSQLError extends ApiError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(ApiErrorType.EXECUTE_SQL, message, errors);
   }
 }
diff --git a/api/src/errors/http-error.ts b/api/src/errors/http-error.ts
@@ -11,9 +11,15 @@ export enum HTTPErrorType {
 
 export class HTTPError extends Error {
   status: number;
-  errors?: (string | object)[];
-
-  constructor(name: HTTPErrorType, status: number, message: string, errors?: (string | object)[], stack?: string) {
+  errors?: (string | Record<string, unknown>)[];
+
+  constructor(
+    name: HTTPErrorType,
+    status: number,
+    message: string,
+    errors?: (string | Record<string, unknown>)[],
+    stack?: string
+  ) {
     super(message);
 
     this.name = name;
@@ -38,7 +44,7 @@ export class HTTPError extends Error {
  * @extends {HTTPError}
  */
 export class HTTP400 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.BAD_REQUEST, 400, message, errors);
   }
 }
@@ -51,7 +57,7 @@ export class HTTP400 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP401 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.UNAUTHORIZE, 401, message, errors);
   }
 }
@@ -64,7 +70,7 @@ export class HTTP401 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP403 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.FORBIDDEN, 403, message, errors);
   }
 }
@@ -77,7 +83,7 @@ export class HTTP403 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP409 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.CONFLICT, 409, message, errors);
   }
 }
@@ -90,7 +96,7 @@ export class HTTP409 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP500 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.INTERNAL_SERVER_ERROR, 500, message, errors);
   }
 }

diff --git a/api/src/openapi/root-api-doc.ts b/api/src/openapi/root-api-doc.ts
@@ -143,60 +143,39 @@ export const rootAPIDoc = {
       },
       SubmissionFeature: {
         title: 'BioHub Data Submission Feature',
+        description:
+          'The submission feature object is a self-referencing recursive structure designed to support different classes of biotic data, in a hierarchical structure.',
         type: 'object',
-        required: ['id', 'type', 'properties', 'features'],
+        required: ['type', 'properties', 'child_features'],
         properties: {
           id: {
-            title: 'Unique id of the feature',
-            type: 'string'
+            title: 'Unique identifer.',
+            description:
+              'The unique identifier for the submission feature as supplied by the source system. May not be unique globally or within BioHub.',
+            type: 'string',
+            maxLength: 200
           },
           type: {
-            title: 'Feature type',
+            title: 'Feature type.',
+            description: 'The type of the feature. Must match a supported feature type.',
             type: 'string'
           },
           properties: {
-            title: 'Feature properties',
+            title: 'Feature properties.',
+            description: 'The properties of the feature, which are specific to the feature type.',
             type: 'object',
             properties: {}
           },
-          features: {
-            title: 'Feature child features',
+          child_features: {
+            title: 'Child features.',
+            description: 'Child features of the current feature.',
             type: 'array',
             items: {
               $ref: '#/components/schemas/SubmissionFeature'
             }
           }
         },
         additionalProperties: false
-      },
-      feature: {
-        type: 'object',
-        required: ['submission_feature_id', 'submission_id', 'feature_type', 'data', 'parent_submission_feature_id'],
-        properties: {
-          submission_feature_id: {
-            type: 'number'
-          },
-          submission_id: {
-            type: 'number'
-          },
-          feature_type: {
-            type: 'string'
-          },
-          data: {
-            type: 'object'
-          },
-          submission_feature_security_ids: {
-            nullable: true,
-            type: 'array',
-            items: {
-              type: 'number'
-            }
-          },
-          parent_submission_feature_id: {
-            type: 'number',
-            nullable: true
-          }
-        }
       }
     }
   }

diff --git a/api/src/openapi/schemas/biohub-data-submission.ts b/api/src/openapi/schemas/biohub-data-submission.ts
diff --git a/api/src/paths/administrative/security/categories.ts b/api/src/paths/administrative/security/categories.ts
@@ -80,13 +80,15 @@ GET.apiDoc = {
                   nullable: true
                 },
                 update_user: {
-                  type: 'string',
+                  type: 'integer',
+                  minimum: 1,
                   nullable: true
                 },
                 revision_count: {
                   type: 'integer'
                 }
-              }
+              },
+              additionalProperties: false
             }
           }
         }

diff --git a/api/src/paths/administrative/security/rules.ts b/api/src/paths/administrative/security/rules.ts
@@ -46,6 +46,7 @@ GET.apiDoc = {
                 'description',
                 'record_effective_date',
                 'record_end_date',
+                'security_category_id',
                 'category_name',
                 'category_description',
                 'category_record_effective_date',
@@ -68,6 +69,10 @@ GET.apiDoc = {
                   type: 'string',
                   nullable: true
                 },
+                security_category_id: {
+                  type: 'integer',
+                  minimum: 1
+                },
                 category_name: {
                   type: 'string'
                 },
@@ -81,7 +86,8 @@ GET.apiDoc = {
                   type: 'string',
                   nullable: true
                 }
-              }
+              },
+              additionalProperties: false
             }
           }
         }

diff --git a/api/src/paths/administrative/security/submission/{submissionId}.ts b/api/src/paths/administrative/security/submission/{submissionId}.ts
@@ -51,18 +51,60 @@ GET.apiDoc = {
             type: 'array',
             items: {
               type: 'object',
-              required: ['submission_feature_security_id', 'submission_feature_id', 'security_rule_id'],
+              required: [
+                'submission_feature_security_id',
+                'submission_feature_id',
+                'security_rule_id',
+                'record_effective_date',
+                'record_end_date',
+                'create_date',
+                'create_user',
+                'update_date',
+                'update_user',
+                'revision_count'
+              ],
               properties: {
                 submission_feature_security_id: {
-                  type: 'integer'
+                  type: 'integer',
+                  minimum: 1
                 },
                 submission_feature_id: {
-                  type: 'integer'
+                  type: 'integer',
+                  minimum: 1
                 },
                 security_rule_id: {
-                  type: 'integer'
+                  type: 'integer',
+                  minimum: 1
+                },
+                record_effective_date: {
+                  type: 'string'
+                },
+                record_end_date: {
+                  type: 'string',
+                  nullable: true
+                },
+                create_date: {
+                  type: 'string'
+                },
+                create_user: {
+                  type: 'integer',
+                  minimum: 1
+                },
+                update_date: {
+                  type: 'string',
+                  nullable: true
+                },
+                update_user: {
+                  type: 'integer',
+                  minimum: 1,
+                  nullable: true
+                },
+                revision_count: {
+                  type: 'integer',
+                  minimum: 0
                 }
-              }
+              },
+              additionalProperties: false
             }
           }
         }