SIMSBIOHUB-424: Support Artifact Intake + Misc Enhancements

api/package.json

@@ -44,7 +44,7 @@
     "fast-xml-parser": "~4.1.3",
     "fastq": "^1.15.0",
     "json-schema-traverse": "^1.0.0",
-    "jsonpath-plus": "~7.2.0",
+    "jsonpath-plus": "^8.0.0",
     "jsonwebtoken": "~8.5.1",
     "jwks-rsa": "~2.0.5",
     "knex": "~2.4.2",

api/src/app.ts

@@ -5,9 +5,10 @@
 import swaggerUIExperss from 'swagger-ui-express';
 import { defaultPoolConfig, initDBPool } from './database/db';
 import { initDBConstants } from './database/db-constants';
-import { ensureHTTPError, HTTPErrorType } from './errors/http-error';
+import { ensureHTTPError, HTTP400, HTTPErrorType } from './errors/http-error';
 import { rootAPIDoc } from './openapi/root-api-doc';
 import { authenticateRequest, authenticateRequestOptional } from './request-handlers/security/authentication';
+import { scanFileForVirus } from './utils/file-utils';
 import { getLogger } from './utils/logger';
 
 const defaultLog = getLogger('app');
@@ -53,22 +54,29 @@
   docsPath: '/raw-api-docs', // path to view raw openapi spec
   consumesMiddleware: {
     'application/json': express.json({ limit: MAX_REQ_BODY_SIZE }),
-    'multipart/form-data': function (req, res, next) {
+    'multipart/form-data': async function (req, res, next) {
       const multerRequestHandler = multer({
-        storage: multer.memoryStorage(),
+        storage: multer.memoryStorage(), // TOOD change to local/PVC storage and stream file uploads to S3?
         limits: { fileSize: MAX_UPLOAD_FILE_SIZE }
       }).array('media', MAX_UPLOAD_NUM_FILES);
 
-      multerRequestHandler(req, res, (error?: any) => {
+      return multerRequestHandler(req, res, async function (error?: any) {
         if (error) {
           return next(error);
         }
 
-        if (req.files && req.files.length) {
+        const promises = (req.files as Express.Multer.File[]).map(async function (file) {
           // Set original request file field to empty string to satisfy OpenAPI validation
           // See: https://www.npmjs.com/package/express-openapi#argsconsumesmiddleware
-          (req.files as Express.Multer.File[]).forEach((file) => (req.body[file.fieldname] = ''));
-        }
+          req.body[file.fieldname] = '';
+
+          // Scan file for malicious content, if enabled
+          if (!(await scanFileForVirus(file))) {
+            throw new HTTP400('Malicious file content detected.', [{ file_name: file.originalname }]);
+          }
+        });
+
+        await Promise.all(promises);
 
         return next();
       });

api/src/database/db-utils.ts

@@ -41,7 +41,7 @@
  */
 const parseError = (error: any) => {
   if (error instanceof z.ZodError) {
-    throw new ApiExecuteSQLError('SQL response failed schema check', [error]);
+    throw new ApiExecuteSQLError('SQL response failed schema check', [error as Record<string, any>]);
   }
 
   if (error.message === 'CONCURRENCY_EXCEPTION') {

api/src/database/db.ts

@@ -392,7 +392,7 @@
 
       _systemUserId = response?.rows?.[0].api_set_context;
     } catch (error) {
-      throw new ApiExecuteSQLError('Failed to set user context', [error as object]);
+      throw new ApiExecuteSQLError('Failed to set user context', [error as Record<string, unknown>]);
     }
   };
 

api/src/errors/api-error.ts

@@ -6,9 +6,9 @@ export enum ApiErrorType {
 }
 
 export class ApiError extends Error {
-  errors?: (string | object)[];
+  errors?: (string | Record<string, unknown>)[];
 
-  constructor(name: ApiErrorType, message: string, errors?: (string | object)[], stack?: string) {
+  constructor(name: ApiErrorType, message: string, errors?: (string | Record<string, unknown>)[], stack?: string) {
     super(message);
 
     this.name = name;
@@ -33,7 +33,7 @@ export class ApiError extends Error {
  * @extends {ApiError}
  */
 export class ApiGeneralError extends ApiError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(ApiErrorType.GENERAL, message, errors);
   }
 }
@@ -46,7 +46,7 @@ export class ApiGeneralError extends ApiError {
  * @extends {ApiError}
  */
 export class ApiUnknownError extends ApiError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(ApiErrorType.UNKNOWN, message, errors);
   }
 }
@@ -63,7 +63,7 @@ export class ApiUnknownError extends ApiError {
  * @extends {ApiError}
  */
 export class ApiExecuteSQLError extends ApiError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(ApiErrorType.EXECUTE_SQL, message, errors);
   }
 }

api/src/errors/http-error.ts

@@ -11,9 +11,15 @@ export enum HTTPErrorType {
 
 export class HTTPError extends Error {
   status: number;
-  errors?: (string | object)[];
-
-  constructor(name: HTTPErrorType, status: number, message: string, errors?: (string | object)[], stack?: string) {
+  errors?: (string | Record<string, unknown>)[];
+
+  constructor(
+    name: HTTPErrorType,
+    status: number,
+    message: string,
+    errors?: (string | Record<string, unknown>)[],
+    stack?: string
+  ) {
     super(message);
 
     this.name = name;
@@ -38,7 +44,7 @@ export class HTTPError extends Error {
  * @extends {HTTPError}
  */
 export class HTTP400 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.BAD_REQUEST, 400, message, errors);
   }
 }
@@ -51,7 +57,7 @@ export class HTTP400 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP401 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.UNAUTHORIZE, 401, message, errors);
   }
 }
@@ -64,7 +70,7 @@ export class HTTP401 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP403 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.FORBIDDEN, 403, message, errors);
   }
 }
@@ -77,7 +83,7 @@ export class HTTP403 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP409 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.CONFLICT, 409, message, errors);
   }
 }
@@ -90,7 +96,7 @@ export class HTTP409 extends HTTPError {
  * @extends {HTTPError}
  */
 export class HTTP500 extends HTTPError {
-  constructor(message: string, errors?: (string | object)[]) {
+  constructor(message: string, errors?: (string | Record<string, unknown>)[]) {
     super(HTTPErrorType.INTERNAL_SERVER_ERROR, 500, message, errors);
   }
 }

api/src/openapi/root-api-doc.ts

@@ -143,60 +143,39 @@ export const rootAPIDoc = {
       },
       SubmissionFeature: {
         title: 'BioHub Data Submission Feature',
+        description:
+          'The submission feature object is a self-referencing recursive structure designed to support different classes of biotic data, in a hierarchical structure.',
         type: 'object',
-        required: ['id', 'type', 'properties', 'features'],
+        required: ['type', 'properties', 'child_features'],
         properties: {
           id: {
-            title: 'Unique id of the feature',
-            type: 'string'
+            title: 'Unique identifer.',
+            description:
+              'The unique identifier for the submission feature as supplied by the source system. May not be unique globally or within BioHub.',
+            type: 'string',
+            maxLength: 200
           },
           type: {
-            title: 'Feature type',
+            title: 'Feature type.',
+            description: 'The type of the feature. Must match a supported feature type.',
             type: 'string'
           },
           properties: {
-            title: 'Feature properties',
+            title: 'Feature properties.',
+            description: 'The properties of the feature, which are specific to the feature type.',
             type: 'object',
             properties: {}
           },
-          features: {
-            title: 'Feature child features',
+          child_features: {
+            title: 'Child features.',
+            description: 'Child features of the current feature.',
             type: 'array',
             items: {
               $ref: '#/components/schemas/SubmissionFeature'
             }
           }
         },
         additionalProperties: false
-      },
-      feature: {
-        type: 'object',
-        required: ['submission_feature_id', 'submission_id', 'feature_type', 'data', 'parent_submission_feature_id'],
-        properties: {
-          submission_feature_id: {
-            type: 'number'
-          },
-          submission_id: {
-            type: 'number'
-          },
-          feature_type: {
-            type: 'string'
-          },
-          data: {
-            type: 'object'
-          },
-          submission_feature_security_ids: {
-            nullable: true,
-            type: 'array',
-            items: {
-              type: 'number'
-            }
-          },
-          parent_submission_feature_id: {
-            type: 'number',
-            nullable: true
-          }
-        }
       }
     }
   }

api/src/paths/administrative/security/categories.ts

@@ -80,13 +80,15 @@ GET.apiDoc = {
                   nullable: true
                 },
                 update_user: {
-                  type: 'string',
+                  type: 'integer',
+                  minimum: 1,
                   nullable: true
                 },
                 revision_count: {
                   type: 'integer'
                 }
-              }
+              },
+              additionalProperties: false
             }
           }
         }

api/src/paths/administrative/security/rules.ts

@@ -46,6 +46,7 @@ GET.apiDoc = {
                 'description',
                 'record_effective_date',
                 'record_end_date',
+                'security_category_id',
                 'category_name',
                 'category_description',
                 'category_record_effective_date',
@@ -68,6 +69,10 @@ GET.apiDoc = {
                   type: 'string',
                   nullable: true
                 },
+                security_category_id: {
+                  type: 'integer',
+                  minimum: 1
+                },
                 category_name: {
                   type: 'string'
                 },
@@ -81,7 +86,8 @@ GET.apiDoc = {
                   type: 'string',
                   nullable: true
                 }
-              }
+              },
+              additionalProperties: false
             }
           }
         }

api/src/paths/administrative/security/submission/{submissionId}.ts

@@ -51,18 +51,60 @@ GET.apiDoc = {
             type: 'array',
             items: {
               type: 'object',
-              required: ['submission_feature_security_id', 'submission_feature_id', 'security_rule_id'],
+              required: [
+                'submission_feature_security_id',
+                'submission_feature_id',
+                'security_rule_id',
+                'record_effective_date',
+                'record_end_date',
+                'create_date',
+                'create_user',
+                'update_date',
+                'update_user',
+                'revision_count'
+              ],
               properties: {
                 submission_feature_security_id: {
-                  type: 'integer'
+                  type: 'integer',
+                  minimum: 1
                 },
                 submission_feature_id: {
-                  type: 'integer'
+                  type: 'integer',
+                  minimum: 1
                 },
                 security_rule_id: {
-                  type: 'integer'
+                  type: 'integer',
+                  minimum: 1
+                },
+                record_effective_date: {
+                  type: 'string'
+                },
+                record_end_date: {
+                  type: 'string',
+                  nullable: true
+                },
+                create_date: {
+                  type: 'string'
+                },
+                create_user: {
+                  type: 'integer',
+                  minimum: 1
+                },
+                update_date: {
+                  type: 'string',
+                  nullable: true
+                },
+                update_user: {
+                  type: 'integer',
+                  minimum: 1,
+                  nullable: true
+                },
+                revision_count: {
+                  type: 'integer',
+                  minimum: 0
                 }
-              }
+              },
+              additionalProperties: false
             }
           }
         }