Edit sampling site fixes

Makefile

@@ -309,7 +309,17 @@ pipeline-install: ## Runs `npm install` for all projects
 ## Run `docker logs <container> -f` commands for all projects
 ## - You can include additional parameters by appaending an `args` param
 ## - Ex: `make log-app args="--tail 0"`
+## Note: The default args, if not provided, are `--tail 2000`
 ## ------------------------------------------------------------------------------
+
+args ?= --tail 2000 ## Default args if none are provided
+
+log: ## Runs `docker-compose logs -f` for all containers
+	@echo "==============================================="
+	@echo "Running docker logs for the app container"
+	@echo "==============================================="
+	@docker-compose logs -f $(args)
+
 log-app: ## Runs `docker logs <container> -f` for the app container
 	@echo "==============================================="
 	@echo "Running docker logs for the app container"

api/src/app.ts

@@ -187,12 +187,12 @@
         return json.apply(res, args);
       }
 
-      const body = args[0];
+      const reqBody = args[0];
 
       // Run openapi response validation function
       const validationResult: { message: any; errors: any[] } | undefined = res['validateResponse'](
         res.statusCode,
-        body
+        reqBody
       );
 
       let validationMessage = '';
@@ -215,7 +215,9 @@
           message: validationMessage,
           error: errorList,
           req_url: `${req.method} ${req.url}`,
-          res_body: body
+          req_params: req.params,
+          req_body: req.body,
+          res_body: reqBody
         });
 
         throw new HTTP500(validationMessage, errorList);

api/src/database/db-utils.test.ts

@@ -1,67 +1,13 @@
 import { expect } from 'chai';
-import { QueryResult } from 'pg';
 import sinon from 'sinon';
-import { z } from 'zod';
 import { SYSTEM_IDENTITY_SOURCE } from '../constants/database';
 import {
   BceidBasicUserInformation,
   BceidBusinessUserInformation,
   DatabaseUserInformation,
   IdirUserInformation
 } from '../utils/keycloak-utils';
-import { getGenericizedKeycloakUserInformation, getZodQueryResult } from './db-utils';
-
-/**
- * Enforces that a zod schema satisfies an existing type definition.
- *
- * Code copied from: https://github.com/colinhacks/zod/issues/372#issuecomment-1280054492
- * An unresolved feature request was opened as well: https://github.com/colinhacks/zod/issues/2084
- *
- * Note: This may not be sufficient to cover ALL possible scenarios.
- *
- * @example
- * const myZodSchema = z.object({...});
- * // Compile error if `myZodSchema` doesn't satisfy `TypeDefinitionToCompareTo`
- * zodImplements<TypeDefinitionToCompareTo>().with(myZodSchema.shape);
- *
- * @template Model
- * @return {*}
- */
-function zodImplements<Model = never>() {
-  type ZodImplements<Model> = {
-    [key in keyof Model]-?: undefined extends Model[key]
-      ? null extends Model[key]
-        ? z.ZodNullableType<z.ZodOptionalType<z.ZodType<Model[key]>>>
-        : z.ZodOptionalType<z.ZodType<Model[key]>>
-      : null extends Model[key]
-      ? z.ZodNullableType<z.ZodType<Model[key]>>
-      : z.ZodType<Model[key]>;
-  };
-
-  return {
-    with: <
-      Schema extends ZodImplements<Model> & {
-        [unknownKey in Exclude<keyof Schema, keyof Model>]: never;
-      }
-    >(
-      schema: Schema
-    ) => z.object(schema)
-  };
-}
-
-describe('getZodQueryResult', () => {
-  it('defines a zod schema that conforms to the real pg `QueryResult` type', () => {
-    const zodQueryResultRow = z.object({});
-
-    const zodQueryResult = getZodQueryResult(zodQueryResultRow);
-
-    // Not a traditional test: will just cause a compile error if the zod schema doesn't satisfy the `QueryResult` type
-    zodImplements<QueryResult>().with(zodQueryResult.shape);
-
-    // Dummy assertion to satisfy linter
-    expect(true).to.be.true;
-  });
-});
+import { getGenericizedKeycloakUserInformation } from './db-utils';
 
 describe('getGenericizedKeycloakUserInformation', () => {
   afterEach(() => {

api/src/database/db-utils.ts

@@ -77,34 +77,6 @@ const parseError = (error: any) => {
   throw new ApiExecuteSQLError('Failed to execute SQL', [error]);
 };
 
-/**
- * A re-definition of the pg `QueryResult` type using Zod.
- *
- * @template T
- * @param {T} zodQueryResultRow A zod schema, used to define the `rows` field of the response. In pg, this would
- * be the `QueryResultRow` type.
- */
-export const getZodQueryResult = <T extends z.Schema>(zodQueryResultRow: T) =>
-  z.object({
-    rows: z.array(zodQueryResultRow),
-    command: z.string(),
-    rowCount: z.number().nullable(),
-    // Using `coerce` as a workaround for an issue with the QueryResult type definition: it specifies oid is always a
-    // number, but in reality it can return `null`.
-    oid: z.coerce.number(),
-    fields: z.array(
-      z.object({
-        name: z.string(),
-        tableID: z.number(),
-        columnID: z.number(),
-        dataTypeID: z.number(),
-        dataTypeSize: z.number(),
-        dataTypeModifier: z.number(),
-        format: z.string()
-      })
-    )
-  });
-
 /**
  * Converts a type specific keycloak user information object with type specific properties into a new object with
  * generic properties.

api/src/database/db.ts

@@ -13,12 +13,7 @@
   ServiceClientUserInformation
 } from '../utils/keycloak-utils';
 import { getLogger } from '../utils/logger';
-import {
-  asyncErrorWrapper,
-  getGenericizedKeycloakUserInformation,
-  getZodQueryResult,
-  syncErrorWrapper
-} from './db-utils';
+import { asyncErrorWrapper, getGenericizedKeycloakUserInformation, syncErrorWrapper } from './db-utils';
 
 const defaultLog = getLogger('database/db');
 
@@ -152,33 +147,37 @@
    * Performs a query against this connection, returning the results.
    *
    * @example
-   * // Create a basic SQLStatement object
    * const sqlStatement = SQL`select * from table where name = ${name}`;
+   * const response = await connection.sql(sqlStatement, ZodSchema);
    *
    * @param {SQLStatement} sqlStatement SQL statement object
-   * @param {z.Schema<T, any, any>} zodSchema An optional zod schema
-   * @return {*}  {(Promise<QueryResult<any>>)}
+   * @param {z.ZodSchema<T, any, any>} [ZodSchema] An optional zod schema that defines the expected shape of a `row`.
+   * @return {*}  {(Promise<QueryResult<T>>)}
    * @throws If the connection is not open.
    * @memberof IDBConnection
    */
   sql: <T extends pg.QueryResultRow = any>(
     sqlStatement: SQLStatement,
-    zodSchema?: z.Schema<T, any, any>
+    ZodSchema?: z.ZodSchema<T, any, any>
   ) => Promise<pg.QueryResult<T>>;
   /**
    * Performs a query against this connection, returning the results.
    *
+   * @example
+   * const queryBuilder = getKnex().select().from('table').where('name', name);
+   * const response = await connection.knex(queryBuilder, ZodSchema);
+   *
    * @see {@link getKnex} to get a knex instance.
    *
    * @param {Knex.QueryBuilder} queryBuilder Knex query builder object
-   * @param {z.Schema<T, any, any>} zodSchema An optional zod schema
-   * @return {*}  {(Promise<QueryResult<any>>)}
+   * @param {z.ZodSchema<T, any, any>} [ZodSchema] An optional zod schema that defines the expected shape of a `row`.
+   * @return {*}  {(Promise<QueryResult<T>>)}
    * @throws If the connection is not open.
    * @memberof IDBConnection
    */
   knex: <T extends pg.QueryResultRow = any>(
     queryBuilder: Knex.QueryBuilder,
-    zodSchema?: z.Schema<T, any, any>
+    ZodSchema?: z.ZodSchema<T, any, any>
   ) => Promise<pg.QueryResult<T>>;
   /**
    * Get the ID of the system user in context.
@@ -341,85 +340,125 @@
    *
    * @template T
    * @param {SQLStatement} sqlStatement SQL statement object
-   * @param {z.Schema<T, any, any>} zodSchema An optional zod schema
+   * @param {z.ZodSchema<T, any, any>} [ZodSchema] An optional zod schema that defines the expected shape of a `row`.
    * @throws {Error} if the connection is not open
    * @return {*}  {Promise<pg.QueryResult<T>>}
    */
   const _sql = async <T extends pg.QueryResultRow = any>(
     sqlStatement: SQLStatement,
-    zodSchema?: z.Schema<T, any, any>
+    ZodSchema?: z.ZodSchema<T, any, any>
   ): Promise<pg.QueryResult<T>> => {
-    if (process.env.DATABASE_RESPONSE_VALIDATION_ENABLED !== 'true') {
-      // Don't validate database responses against provided zod schema
-      return _query(sqlStatement.text, sqlStatement.values);
-    }
-
     const queryStart = Date.now();
+
     const response = await _query(sqlStatement.text, sqlStatement.values);
+
     const queryEnd = Date.now();
 
-    if (!zodSchema) {
-      defaultLog.silly({ label: '_sql', message: sqlStatement.text, queryExecutionTime: queryEnd - queryStart });
-      // No zod schema provided
+    defaultLog.silly({
+      label: '_sql',
+      message: 'Sql performance',
+      sql: { sql: sqlStatement.text, bindings: sqlStatement.values },
+      queryExecutionTime: queryEnd - queryStart
+    });
+
+    if (!ZodSchema || process.env.DATABASE_RESPONSE_VALIDATION_ENABLED !== 'true') {
+      // No zod schema provided, or database response validation is disabled
       return response;
     }
 
-    // Validate the response against the zod schema
+    // Validate the response rows against the zod schema
     const zodStart = Date.now();
-    const validatedResponse = getZodQueryResult(zodSchema).parseAsync(response);
+
+    const zodResponse =
+      ZodSchema instanceof z.ZodObject
+        ? z.strictObject({ rows: z.array(ZodSchema.strict()) }).safeParse({ rows: response.rows })
+        : z.strictObject({ rows: z.array(ZodSchema) }).safeParse({ rows: response.rows });
+
     const zodEnd = Date.now();
 
     defaultLog.silly({
-      label: '_sql + zod',
-      message: sqlStatement.text,
+      label: '_sql',
+      message: 'Zod performance',
+      sql: { sql: sqlStatement.text, bindings: sqlStatement.values },
       queryExecutionTime: queryEnd - queryStart,
       zodExecutionTime: zodEnd - zodStart
     });
-    return validatedResponse;
+
+    if (!zodResponse.success) {
+      defaultLog.debug({
+        label: '_sql',
+        message: 'zodResponse',
+        zodResponse
+      });
+
+      throw new ApiExecuteSQLError('Failed to validate database response', zodResponse.error.errors);
+    }
+
+    return response;
   };
 
   /**
    * Performs a query against this connection, returning the results.
    *
    * @template T
    * @param {Knex.QueryBuilder} queryBuilder Knex query builder object
-   * @param {z.Schema<T, any, any>} zodSchema An optional zod schema
+   * @param {z.ZodSchema<T, any, any>} [ZodSchema] An optional zod schema that defines the expected shape of a `row`.
    * @throws {Error} if the connection is not open
    * @return {*}  {Promise<pg.QueryResult<T>>}
    */
   const _knex = async <T extends pg.QueryResultRow = any>(
     queryBuilder: Knex.QueryBuilder,
-    zodSchema?: z.Schema<T, any, any>
+    ZodSchema?: z.ZodSchema<T, any, any>
   ) => {
     const { sql, bindings } = queryBuilder.toSQL().toNative();
 
-    if (process.env.DATABASE_RESPONSE_VALIDATION_ENABLED !== 'true') {
-      // Don't validate database responses against provided zod schema
-      return _query(sql, bindings as any[]);
-    }
-
     const queryStart = Date.now();
+
     const response = await _query(sql, bindings as any[]);
+
     const queryEnd = Date.now();
 
-    if (!zodSchema) {
-      defaultLog.silly({ label: '_knex', message: sql, queryExecutionTime: queryEnd - queryStart });
-      // No zod schema provided
+    defaultLog.silly({
+      label: '_knex',
+      message: 'Sql performance',
+      sql: { sql, bindings },
+      queryExecutionTime: queryEnd - queryStart
+    });
+
+    if (!ZodSchema || process.env.DATABASE_RESPONSE_VALIDATION_ENABLED !== 'true') {
+      // No zod schema provided, or database response validation is disabled
       return response;
     }
 
-    // Validate the response against the zod schema
+    // Validate the response rows against the zod schema
     const zodStart = Date.now();
-    const validatedResponse = getZodQueryResult(zodSchema).parseAsync(response);
+
+    const zodResponse =
+      ZodSchema instanceof z.ZodObject
+        ? z.strictObject({ rows: z.array(ZodSchema.strict()) }).safeParse({ rows: response.rows })
+        : z.object({ rows: z.array(ZodSchema) }).safeParse({ rows: response.rows });
+
     const zodEnd = Date.now();
 
     defaultLog.silly({
-      label: '_knex + zod',
-      message: sql,
+      label: '_knex',
+      message: 'Zod performance',
+      sql: { sql, bindings },
       queryExecutionTime: queryEnd - queryStart,
       zodExecutionTime: zodEnd - zodStart
     });
-    return validatedResponse;
+
+    if (!zodResponse.success) {
+      defaultLog.debug({
+        label: '_knex',
+        message: 'zodResponse',
+        zodResponse
+      });
+
+      throw new ApiExecuteSQLError('Failed to validate database response', zodResponse.error.errors);
+    }
+
+    return response;
   };
 
   /**

api/src/models/biohub-create.test.ts

@@ -16,6 +16,7 @@ describe('PostSurveyObservationToBiohubObject', () => {
     const obj = {
       survey_observation_id: 1,
       survey_id: 1,
+      wldtaxonomic_units_id: 1,
       survey_sample_site_id: 1,
       survey_sample_method_id: 1,
       survey_sample_period_id: 1,
@@ -158,6 +159,7 @@ describe('PostSurveySubmissionToBioHubObject', () => {
       {
         survey_observation_id: 1,
         survey_id: 1,
+        wldtaxonomic_units_id: 1,
         survey_sample_site_id: 1,
         survey_sample_method_id: 1,
         survey_sample_period_id: 1,