From bb4f7dd5dfe1e507fcf9194fd6fbb54f9fed1224 Mon Sep 17 00:00:00 2001 From: Macgregor Aubertin-Young <108430771+mauberti-bc@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:29:42 -0700 Subject: [PATCH 1/3] Update styling for the Add User form (#1355) * update styling of add user form * fix tests * fix: tweaked some props for edit dialog --------- Co-authored-by: Mac Deluca <99926243+MacQSL@users.noreply.github.com> Co-authored-by: Mac Deluca --- app/src/AppRouter.tsx | 4 +- app/src/components/fields/CustomTextField.tsx | 10 +- .../admin/users/ActiveUsersList.test.tsx | 20 +- .../features/admin/users/ActiveUsersList.tsx | 64 +++-- .../admin/users/AddSystemUsersForm.tsx | 253 +++++++----------- .../admin/users/ManageUsersPage.test.tsx | 18 +- 6 files changed, 178 insertions(+), 191 deletions(-) diff --git a/app/src/AppRouter.tsx b/app/src/AppRouter.tsx index 80bbebf558..5e09060113 100644 --- a/app/src/AppRouter.tsx +++ b/app/src/AppRouter.tsx @@ -83,7 +83,9 @@ const AppRouter: React.FC = () => { - + + + diff --git a/app/src/components/fields/CustomTextField.tsx b/app/src/components/fields/CustomTextField.tsx index 38744291fd..4f50ed4ae3 100644 --- a/app/src/components/fields/CustomTextField.tsx +++ b/app/src/components/fields/CustomTextField.tsx @@ -9,6 +9,13 @@ export interface ICustomTextField { * @memberof ICustomTextField */ label: string; + /** + * Placeholder for the text field + * + * @type {string} + * @memberof ICustomTextField + */ + placeholder?: string; /** * Name of the text field, typically this is used to identify the field in the formik context. * @@ -34,13 +41,14 @@ export interface ICustomTextField { const CustomTextField = (props: React.PropsWithChildren) => { const { touched, errors, values, handleChange, handleBlur } = useFormikContext(); - const { name, label, other } = props; + const { name, label, other, placeholder } = props; return ( {} + } as DataLoader +}; + const renderContainer = (props: IActiveUsersListProps) => { const authState = getMockAuthState({ base: SystemAdminAuthState }); return render( - - - + + + + + ); }; diff --git a/app/src/features/admin/users/ActiveUsersList.tsx b/app/src/features/admin/users/ActiveUsersList.tsx index d6e48eabd0..f8ef7845d7 100644 --- a/app/src/features/admin/users/ActiveUsersList.tsx +++ b/app/src/features/admin/users/ActiveUsersList.tsx @@ -16,11 +16,13 @@ import { DialogContext, ISnackbarProps } from 'contexts/dialogContext'; import { APIError } from 'hooks/api/useAxios'; import { useAuthStateContext } from 'hooks/useAuthStateContext'; import { useBiohubApi } from 'hooks/useBioHubApi'; +import { useCodesContext } from 'hooks/useContext'; import { IGetAllCodeSetsResponse } from 'interfaces/useCodesApi.interface'; import { ISystemUser } from 'interfaces/useUserApi.interface'; -import { useContext, useState } from 'react'; +import { useContext, useEffect, useState } from 'react'; import { useHistory } from 'react-router'; import { Link as RouterLink } from 'react-router-dom'; +import { getCodesName } from 'utils/Utils'; import AddSystemUsersForm, { AddSystemUsersFormInitialValues, AddSystemUsersFormYupSchema, @@ -50,6 +52,12 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { const [openAddUserDialog, setOpenAddUserDialog] = useState(false); + const codesContext = useCodesContext(); + + useEffect(() => { + codesContext.codesDataLoader.load(); + }, [codesContext.codesDataLoader]); + const activeUsersColumnDefs: GridColDef[] = [ { field: 'user_identifier', @@ -279,16 +287,16 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { const handleAddSystemUsersSave = async (values: IAddSystemUsersForm) => { setOpenAddUserDialog(false); + const systemUser = values.systemUser; + try { - for (const systemUser of values.systemUsers) { - await biohubApi.admin.addSystemUser( - systemUser.userIdentifier, - systemUser.identitySource, - systemUser.displayName, - systemUser.email, - systemUser.systemRole - ); - } + await biohubApi.admin.addSystemUser( + systemUser.userIdentifier, + systemUser.identitySource, + systemUser.displayName, + systemUser.email, + systemUser.systemRole + ); // Refresh users list refresh(); @@ -297,7 +305,7 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { open: true, snackbarMessage: ( - {values.systemUsers.length} system {values.systemUsers.length > 1 ? 'users' : 'user'} added. + Successfully added {systemUser.displayName} ) }); @@ -307,8 +315,12 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { if (apiError.status === 409) { dialogContext.setErrorDialog({ open: true, - dialogTitle: 'Failed to create users', - dialogText: 'One of the users you added already exists.', + dialogTitle: 'User already exists', + dialogText: `${systemUser.displayName} already exists as a ${getCodesName( + codesContext.codesDataLoader.data, + 'system_roles', + systemUser.systemRole + )}`, onClose: () => { dialogContext.setErrorDialog({ open: false }); }, @@ -380,21 +392,29 @@ const ActiveUsersList = (props: IActiveUsersListProps) => { { - return { value: item.id, label: item.name }; - }) || [] - } - /> + <> + + This form creates a new user that will be linked to an IDIR/BCeID when an account with a matching + username, email, and account type logs in. + + { + return { value: item.id, label: item.name }; + }) || [] + } + /> + ), initialValues: AddSystemUsersFormInitialValues, - validationSchema: AddSystemUsersFormYupSchema + validationSchema: AddSystemUsersFormYupSchema, + validateOnBlur: false }} onCancel={() => setOpenAddUserDialog(false)} onSave={(values) => { diff --git a/app/src/features/admin/users/AddSystemUsersForm.tsx b/app/src/features/admin/users/AddSystemUsersForm.tsx index 02fafd2ba8..2f13b1d813 100644 --- a/app/src/features/admin/users/AddSystemUsersForm.tsx +++ b/app/src/features/admin/users/AddSystemUsersForm.tsx @@ -1,17 +1,9 @@ -import { mdiPlus, mdiTrashCanOutline } from '@mdi/js'; -import Icon from '@mdi/react'; import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import FormControl from '@mui/material/FormControl'; -import FormHelperText from '@mui/material/FormHelperText'; -import Grid from '@mui/material/Grid'; -import IconButton from '@mui/material/IconButton'; -import InputLabel from '@mui/material/InputLabel'; -import MenuItem from '@mui/material/MenuItem'; -import Select from '@mui/material/Select'; +import Stack from '@mui/material/Stack'; +import AutocompleteField, { IAutocompleteFieldOption } from 'components/fields/AutocompleteField'; import CustomTextField from 'components/fields/CustomTextField'; import { SYSTEM_IDENTITY_SOURCE } from 'constants/auth'; -import { FieldArray, FieldArrayRenderProps, useFormikContext } from 'formik'; +import { useFormikContext } from 'formik'; import React from 'react'; import yup from 'utils/YupSchema'; @@ -24,7 +16,7 @@ export interface IAddSystemUsersFormArrayItem { } export interface IAddSystemUsersForm { - systemUsers: IAddSystemUsersFormArrayItem[]; + systemUser: IAddSystemUsersFormArrayItem; } export const AddSystemUsersFormArrayItemInitialValues: IAddSystemUsersFormArrayItem = { @@ -36,168 +28,107 @@ export const AddSystemUsersFormArrayItemInitialValues: IAddSystemUsersFormArrayI }; export const AddSystemUsersFormInitialValues: IAddSystemUsersForm = { - systemUsers: [AddSystemUsersFormArrayItemInitialValues] + systemUser: AddSystemUsersFormArrayItemInitialValues }; export const AddSystemUsersFormYupSchema = yup.object().shape({ - systemUsers: yup.array().of( - yup.object().shape({ - userIdentifier: yup.string().required('Username is required'), - displayName: yup.string().required('Display Name is required'), - email: yup.string().email('Must be a valid email').required('Email is required'), - identitySource: yup.string().required('Login Method is required'), - systemRole: yup.number().required('Role is required') - }) - ) + systemUser: yup.object().shape({ + userIdentifier: yup.string().required('Username is required'), + displayName: yup.string().required('Display Name is required'), + email: yup.string().email('Must be a valid email').required('Email is required'), + identitySource: yup.string().required('Account Type is required'), + systemRole: yup.number().required('System Role is required') + }) }); export interface AddSystemUsersFormProps { - systemRoles: any[]; + systemRoles: IAutocompleteFieldOption[]; } +/** + * Returns form component for manually adding system users before access is requested + * + * @param props + * @returns + */ const AddSystemUsersForm: React.FC = (props) => { - const { values, handleChange, handleSubmit, getFieldMeta } = useFormikContext(); + const { values, handleSubmit, getFieldMeta } = useFormikContext(); + + const userIdentifierMeta = getFieldMeta('systemUser.userIdentifier'); + const displayNameMeta = getFieldMeta('systemUser.displayName'); + const emailMeta = getFieldMeta('systemUser.email'); + + const { systemRoles } = props; + const identitySources: IAutocompleteFieldOption[] = [ + { value: SYSTEM_IDENTITY_SOURCE.IDIR as string, label: SYSTEM_IDENTITY_SOURCE.IDIR }, + { value: SYSTEM_IDENTITY_SOURCE.BCEID_BASIC as string, label: SYSTEM_IDENTITY_SOURCE.BCEID_BASIC }, + { value: SYSTEM_IDENTITY_SOURCE.BCEID_BUSINESS as string, label: SYSTEM_IDENTITY_SOURCE.BCEID_BUSINESS }, + { value: SYSTEM_IDENTITY_SOURCE.UNVERIFIED as string, label: SYSTEM_IDENTITY_SOURCE.UNVERIFIED } + ]; return (
- ( - - - {values.systemUsers?.map((systemUser: IAddSystemUsersFormArrayItem, index: number) => { - const userIdentifierMeta = getFieldMeta(`systemUsers.[${index}].userIdentifier`); - const displayNameMeta = getFieldMeta(`systemUsers.[${index}].displayName`); - const emailMeta = getFieldMeta(`systemUsers.[${index}].email`); - const identitySourceMeta = getFieldMeta(`systemUsers.[${index}].identitySource`); - const systemRoleMeta = getFieldMeta(`systemUsers.[${index}].roleId`); - - return ( - - - - - - - - - - - - - - - - Login Method - - - {identitySourceMeta.touched && identitySourceMeta.error} - - - - - - System Role - - - {systemRoleMeta.touched && systemRoleMeta.error} - - - - - arrayHelpers.remove(index)} - sx={{ - marginTop: '8px' - }}> - - - - - - ); - })} - - - - + + + + - )} - /> + + + + + + + + + + + + ); }; diff --git a/app/src/features/admin/users/ManageUsersPage.test.tsx b/app/src/features/admin/users/ManageUsersPage.test.tsx index 3cd33589cc..c9d44764ba 100644 --- a/app/src/features/admin/users/ManageUsersPage.test.tsx +++ b/app/src/features/admin/users/ManageUsersPage.test.tsx @@ -1,8 +1,11 @@ import { AuthStateContext } from 'contexts/authStateContext'; +import { CodesContext, ICodesContext } from 'contexts/codesContext'; import { createMemoryHistory } from 'history'; import { useBiohubApi } from 'hooks/useBioHubApi'; +import { DataLoader } from 'hooks/useDataLoader'; import { Router } from 'react-router'; import { getMockAuthState, SystemAdminAuthState } from 'test-helpers/auth-helpers'; +import { codes } from 'test-helpers/code-helpers'; import { cleanup, render, waitFor } from 'test-helpers/test-utils'; import ManageUsersPage from './ManageUsersPage'; @@ -11,11 +14,20 @@ const history = createMemoryHistory(); const renderContainer = () => { const authState = getMockAuthState({ base: SystemAdminAuthState }); + const mockCodesContext: ICodesContext = { + codesDataLoader: { + data: codes, + load: () => {} + } as DataLoader + }; + return render( - - - + + + + + ); }; From 981ee0c54cbb0048eb4da88ab6e985f00f7dd6ee Mon Sep 17 00:00:00 2001 From: Mac Deluca <99926243+MacQSL@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:32:23 -0700 Subject: [PATCH 2/3] SIMSBIOHUB 587: API Measurements (#1344) - Bulk measurements endpoint --- api/.docker/api/Dockerfile | 3 + .../critters/measurements/import.ts | 161 +++++ api/src/services/critterbase-service.ts | 4 +- .../capture/import-captures-strategy.ts | 44 +- .../critter/import-critters-strategy.ts | 14 +- .../import-services/import-csv.interface.ts | 5 + .../marking/import-markings-strategy.ts | 28 +- .../import-measurements-strategy.interface.ts | 22 + .../import-measurements-strategy.test.ts | 650 ++++++++++++++++++ .../import-measurements-strategy.ts | 344 +++++++++ api/src/services/observation-service.ts | 20 +- .../xlsx-utils/column-validator-utils.test.ts | 20 +- .../xlsx-utils/column-validator-utils.ts | 27 +- api/src/utils/xlsx-utils/worksheet-utils.ts | 6 +- 14 files changed, 1272 insertions(+), 76 deletions(-) create mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts create mode 100644 api/src/services/import-services/measurement/import-measurements-strategy.interface.ts create mode 100644 api/src/services/import-services/measurement/import-measurements-strategy.test.ts create mode 100644 api/src/services/import-services/measurement/import-measurements-strategy.ts diff --git a/api/.docker/api/Dockerfile b/api/.docker/api/Dockerfile index 2c658d84b8..a5052b22ce 100644 --- a/api/.docker/api/Dockerfile +++ b/api/.docker/api/Dockerfile @@ -21,6 +21,9 @@ ENV PATH ${HOME}/node_modules/.bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/us # Copy the rest of the files COPY . ./ +# Update log directory file permissions, prevents permission errors for linux environments +RUN chmod -R a+rw data/logs/* + VOLUME ${HOME} # start api with live reload diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts new file mode 100644 index 0000000000..9988b6bb00 --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/critters/measurements/import.ts @@ -0,0 +1,161 @@ +import { RequestHandler } from 'express'; +import { Operation } from 'express-openapi'; +import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles'; +import { getDBConnection } from '../../../../../../../database/db'; +import { HTTP400 } from '../../../../../../../errors/http-error'; +import { csvFileSchema } from '../../../../../../../openapi/schemas/file'; +import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; +import { importCSV } from '../../../../../../../services/import-services/import-csv'; +import { ImportMeasurementsStrategy } from '../../../../../../../services/import-services/measurement/import-measurements-strategy'; +import { scanFileForVirus } from '../../../../../../../utils/file-utils'; +import { getLogger } from '../../../../../../../utils/logger'; +import { parseMulterFile } from '../../../../../../../utils/media/media-utils'; +import { getFileFromRequest } from '../../../../../../../utils/request'; + +const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/measurements/import'); + +export const POST: Operation = [ + authorizeRequestHandler((req) => { + return { + or: [ + { + validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR], + surveyId: Number(req.params.surveyId), + discriminator: 'ProjectPermission' + }, + { + validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + importCsv() +]; + +POST.apiDoc = { + description: 'Upload Critterbase CSV Measurements file', + tags: ['observations'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + in: 'path', + description: 'SIMS survey id', + name: 'projectId', + required: true, + schema: { + type: 'integer', + minimum: 1 + } + }, + { + in: 'path', + description: 'SIMS survey id', + name: 'surveyId', + required: true, + schema: { + type: 'integer', + minimum: 1 + } + } + ], + requestBody: { + description: 'Critterbase Measurements CSV import file.', + content: { + 'multipart/form-data': { + schema: { + type: 'object', + additionalProperties: false, + required: ['media'], + properties: { + media: { + description: 'Critterbase Measurements CSV import file.', + type: 'array', + minItems: 1, + maxItems: 1, + items: csvFileSchema + } + } + } + } + } + }, + responses: { + 201: { + description: 'Measurement import success.', + content: { + 'application/json': { + schema: { + type: 'object', + additionalProperties: false, + properties: { + measurementsCreated: { + description: 'Number of Critterbase measurements created.', + type: 'integer' + } + } + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 403: { + $ref: '#/components/responses/403' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Imports a `Critterbase Measurement CSV` which bulk adds measurements to Critterbase. + * + * @return {*} {RequestHandler} + */ +export function importCsv(): RequestHandler { + return async (req, res) => { + const surveyId = Number(req.params.surveyId); + const rawFile = getFileFromRequest(req); + + const connection = getDBConnection(req.keycloak_token); + + try { + await connection.open(); + + // Check for viruses / malware + const virusScanResult = await scanFileForVirus(rawFile); + + if (!virusScanResult) { + throw new HTTP400('Malicious content detected, import cancelled.'); + } + + const importCsvMeasurementsStrategy = new ImportMeasurementsStrategy(connection, surveyId); + + // Pass CSV file and importer as dependencies + const measurementsCreated = await importCSV(parseMulterFile(rawFile), importCsvMeasurementsStrategy); + + await connection.commit(); + + return res.status(201).json({ measurementsCreated }); + } catch (error) { + defaultLog.error({ label: 'importMeasurementsCSV', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/services/critterbase-service.ts b/api/src/services/critterbase-service.ts index 0a0e7485a2..f76067ad21 100644 --- a/api/src/services/critterbase-service.ts +++ b/api/src/services/critterbase-service.ts @@ -178,8 +178,8 @@ export interface IQualMeasurement { capture_id?: string; mortality_id?: string; qualitative_option_id: string; - measurement_comment: string; - measured_timestamp: string; + measurement_comment?: string; + measured_timestamp?: string; } export interface IQuantMeasurement { diff --git a/api/src/services/import-services/capture/import-captures-strategy.ts b/api/src/services/import-services/capture/import-captures-strategy.ts index e238ec59e3..f69ab1469e 100644 --- a/api/src/services/import-services/capture/import-captures-strategy.ts +++ b/api/src/services/import-services/capture/import-captures-strategy.ts @@ -2,7 +2,7 @@ import { v4 as uuid } from 'uuid'; import { z } from 'zod'; import { IDBConnection } from '../../../database/db'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { ICapture, ILocation } from '../../critterbase-service'; import { DBService } from '../../db-service'; @@ -64,7 +64,7 @@ export class ImportCapturesStrategy extends DBService implements CSVImportStrate */ async validateRows(rows: Row[]) { // Generate type-safe cell getter from column validator - const getCellValue = generateCellGetterFromColumnValidator(this.columnValidator); + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); const critterAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); const rowsToValidate = []; @@ -72,24 +72,24 @@ export class ImportCapturesStrategy extends DBService implements CSVImportStrate for (const row of rows) { let critterId, captureId; - const alias = getCellValue(row, 'ALIAS'); + const alias = getColumnCell(row, 'ALIAS'); - const releaseLatitude = getCellValue(row, 'RELEASE_LATITUDE'); - const releaseLongitude = getCellValue(row, 'RELEASE_LONGITUDE'); - const captureDate = getCellValue(row, 'CAPTURE_DATE'); - const captureTime = getCellValue(row, 'CAPTURE_TIME'); - const releaseTime = getCellValue(row, 'RELEASE_TIME'); + const releaseLatitude = getColumnCell(row, 'RELEASE_LATITUDE'); + const releaseLongitude = getColumnCell(row, 'RELEASE_LONGITUDE'); + const captureDate = getColumnCell(row, 'CAPTURE_DATE'); + const captureTime = getColumnCell(row, 'CAPTURE_TIME'); + const releaseTime = getColumnCell(row, 'RELEASE_TIME'); - const releaseLocationId = releaseLatitude && releaseLongitude ? uuid() : undefined; - const formattedCaptureTime = formatTimeString(captureTime); - const formattedReleaseTime = formatTimeString(releaseTime); + const releaseLocationId = releaseLatitude.cell && releaseLongitude.cell ? uuid() : undefined; + const formattedCaptureTime = formatTimeString(captureTime.cell); + const formattedReleaseTime = formatTimeString(releaseTime.cell); // If the alias is included attempt to retrieve the critterId from row // Checks if date time fields are unique for the critter's captures - if (alias) { - const critter = critterAliasMap.get(alias.toLowerCase()); + if (alias.cell) { + const critter = critterAliasMap.get(alias.cell.toLowerCase()); if (critter) { - const captures = findCapturesFromDateTime(critter.captures, captureDate, captureTime); + const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); critterId = critter.critter_id; // Only set the captureId if a capture does not exist with matching date time captureId = captures.length > 0 ? undefined : uuid(); @@ -100,17 +100,17 @@ export class ImportCapturesStrategy extends DBService implements CSVImportStrate capture_id: captureId, // this will be undefined if capture exists with same date / time critter_id: critterId, capture_location_id: uuid(), - capture_date: captureDate, + capture_date: captureDate.cell, capture_time: formattedCaptureTime, - capture_latitude: getCellValue(row, 'CAPTURE_LATITUDE'), - capture_longitude: getCellValue(row, 'CAPTURE_LONGITUDE'), + capture_latitude: getColumnCell(row, 'CAPTURE_LATITUDE').cell, + capture_longitude: getColumnCell(row, 'CAPTURE_LONGITUDE').cell, release_location_id: releaseLocationId, - release_date: getCellValue(row, 'RELEASE_DATE'), + release_date: getColumnCell(row, 'RELEASE_DATE').cell, release_time: formattedReleaseTime, - release_latitude: getCellValue(row, 'RELEASE_LATITUDE'), - release_longitude: getCellValue(row, 'RELEASE_LONGITUDE'), - capture_comment: getCellValue(row, 'CAPTURE_COMMENT'), - release_comment: getCellValue(row, 'RELEASE_COMMENT') + release_latitude: getColumnCell(row, 'RELEASE_LATITUDE').cell, + release_longitude: getColumnCell(row, 'RELEASE_LONGITUDE').cell, + capture_comment: getColumnCell(row, 'CAPTURE_COMMENT').cell, + release_comment: getColumnCell(row, 'RELEASE_COMMENT').cell }); } diff --git a/api/src/services/import-services/critter/import-critters-strategy.ts b/api/src/services/import-services/critter/import-critters-strategy.ts index 4174fb30f4..f1695e87c0 100644 --- a/api/src/services/import-services/critter/import-critters-strategy.ts +++ b/api/src/services/import-services/critter/import-critters-strategy.ts @@ -5,7 +5,7 @@ import { IDBConnection } from '../../../database/db'; import { ApiGeneralError } from '../../../errors/api-error'; import { getLogger } from '../../../utils/logger'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { getNonStandardColumnNamesFromWorksheet, IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { CritterbaseService, @@ -181,17 +181,17 @@ export class ImportCrittersStrategy extends DBService implements CSVImportStrate * @returns {PartialCsvCritter[]} CSV critters before validation */ _getRowsToValidate(rows: Row[], collectionUnitColumns: string[]): PartialCsvCritter[] { - const getCellValue = generateCellGetterFromColumnValidator(this.columnValidator); + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); return rows.map((row) => { // Standard critter properties from CSV const standardCritterRow = { critter_id: uuid(), // Generate a uuid for each critter for convienence - sex: getCellValue(row, 'SEX'), - itis_tsn: getCellValue(row, 'ITIS_TSN'), - wlh_id: getCellValue(row, 'WLH_ID'), - animal_id: getCellValue(row, 'ALIAS'), - critter_comment: getCellValue(row, 'DESCRIPTION') + sex: getColumnCell(row, 'SEX').cell, + itis_tsn: getColumnCell(row, 'ITIS_TSN').cell, + wlh_id: getColumnCell(row, 'WLH_ID').cell, + animal_id: getColumnCell(row, 'ALIAS').cell, + critter_comment: getColumnCell(row, 'DESCRIPTION').cell }; // All other properties must be collection units ie: `population unit` or `herd unit` etc... diff --git a/api/src/services/import-services/import-csv.interface.ts b/api/src/services/import-services/import-csv.interface.ts index 43c2faf622..6b843ab9e9 100644 --- a/api/src/services/import-services/import-csv.interface.ts +++ b/api/src/services/import-services/import-csv.interface.ts @@ -67,6 +67,11 @@ export type ValidationError = { * */ row: number; + /** + * CSV column header + * + */ + col?: string; /** * CSV row error message * diff --git a/api/src/services/import-services/marking/import-markings-strategy.ts b/api/src/services/import-services/marking/import-markings-strategy.ts index e1fa92f78a..97dbf52237 100644 --- a/api/src/services/import-services/marking/import-markings-strategy.ts +++ b/api/src/services/import-services/marking/import-markings-strategy.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { IDBConnection } from '../../../database/db'; import { getLogger } from '../../../utils/logger'; import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; import { IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; import { IAsSelectLookup, ICritterDetailed } from '../../critterbase-service'; import { DBService } from '../../db-service'; @@ -98,7 +98,7 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportStrate */ async validateRows(rows: Row[]) { // Generate type-safe cell getter from column validator - const getCellValue = generateCellGetterFromColumnValidator(this.columnValidator); + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); // Get validation reference data const [critterAliasMap, colours, markingTypes] = await Promise.all([ @@ -116,18 +116,18 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportStrate for (const row of rows) { let critterId, captureId; - const alias = getCellValue(row, 'ALIAS'); + const alias = getColumnCell(row, 'ALIAS'); // If the alias is included attempt to retrieve the critter_id and capture_id for the row - if (alias) { - const captureDate = getCellValue(row, 'CAPTURE_DATE'); - const captureTime = getCellValue(row, 'CAPTURE_TIME'); + if (alias.cell) { + const captureDate = getColumnCell(row, 'CAPTURE_DATE'); + const captureTime = getColumnCell(row, 'CAPTURE_TIME'); - const critter = critterAliasMap.get(alias.toLowerCase()); + const critter = critterAliasMap.get(alias.cell.toLowerCase()); if (critter) { // Find the capture_id from the date time columns - const captures = findCapturesFromDateTime(critter.captures, captureDate, captureTime); + const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); captureId = captures.length === 1 ? captures[0].capture_id : undefined; critterId = critter.critter_id; rowCritters.push(critter); @@ -137,12 +137,12 @@ export class ImportMarkingsStrategy extends DBService implements CSVImportStrate rowsToValidate.push({ critter_id: critterId, // Found using alias capture_id: captureId, // Found using capture date and time - body_location: getCellValue(row, 'BODY_LOCATION'), - marking_type: getCellValue(row, 'MARKING_TYPE'), - identifier: getCellValue(row, 'IDENTIFIER'), - primary_colour: getCellValue(row, 'PRIMARY_COLOUR'), - secondary_colour: getCellValue(row, 'SECONDARY_COLOUR'), - comment: getCellValue(row, 'DESCRIPTION') + body_location: getColumnCell(row, 'BODY_LOCATION').cell, + marking_type: getColumnCell(row, 'MARKING_TYPE').cell, + identifier: getColumnCell(row, 'IDENTIFIER').cell, + primary_colour: getColumnCell(row, 'PRIMARY_COLOUR').cell, + secondary_colour: getColumnCell(row, 'SECONDARY_COLOUR').cell, + comment: getColumnCell(row, 'DESCRIPTION').cell }); } // Get the critter_id -> taxonBodyLocations[] Map diff --git a/api/src/services/import-services/measurement/import-measurements-strategy.interface.ts b/api/src/services/import-services/measurement/import-measurements-strategy.interface.ts new file mode 100644 index 0000000000..2cfabf887f --- /dev/null +++ b/api/src/services/import-services/measurement/import-measurements-strategy.interface.ts @@ -0,0 +1,22 @@ +import { z } from 'zod'; + +export const CsvQualitativeMeasurementSchema = z.object({ + critter_id: z.string().uuid(), + capture_id: z.string().uuid(), + taxon_measurement_id: z.string().uuid(), + qualitative_option_id: z.string().uuid() +}); + +export const CsvQuantitativeMeasurementSchema = z.object({ + critter_id: z.string().uuid(), + capture_id: z.string().uuid(), + taxon_measurement_id: z.string().uuid(), + value: z.number() +}); + +export const CsvMeasurementSchema = CsvQualitativeMeasurementSchema.or(CsvQuantitativeMeasurementSchema); + +// Zod inferred types +export type CsvMeasurement = z.infer; +export type CsvQuantitativeMeasurement = z.infer; +export type CsvQualitativeMeasurement = z.infer; diff --git a/api/src/services/import-services/measurement/import-measurements-strategy.test.ts b/api/src/services/import-services/measurement/import-measurements-strategy.test.ts new file mode 100644 index 0000000000..9503e0569c --- /dev/null +++ b/api/src/services/import-services/measurement/import-measurements-strategy.test.ts @@ -0,0 +1,650 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { MediaFile } from '../../../utils/media/media-file'; +import * as worksheetUtils from '../../../utils/xlsx-utils/worksheet-utils'; +import { getMockDBConnection } from '../../../__mocks__/db'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition, + IBulkCreateResponse +} from '../../critterbase-service'; +import { importCSV } from '../import-csv'; +import { ImportMeasurementsStrategy } from './import-measurements-strategy'; + +describe('importMeasurementsStrategy', () => { + describe('importCSV', () => { + beforeEach(() => { + sinon.restore(); + }); + + it('should import the csv file correctly', async () => { + const worksheet = { + A1: { t: 's', v: 'ALIAS' }, + B1: { t: 's', v: 'CAPTURE_DATE' }, + C1: { t: 's', v: 'CAPTURE_TIME' }, + D1: { t: 's', v: 'tail length' }, + E1: { t: 's', v: 'skull condition' }, + F1: { t: 's', v: 'unknown' }, + A2: { t: 's', v: 'carl' }, + B2: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' }, + C2: { t: 's', v: '10:10:12' }, + D2: { t: 'n', w: '2', v: 2 }, + E2: { t: 'n', w: '0', v: 'good' }, + A3: { t: 's', v: 'carlita' }, + B3: { z: 'm/d/yy', t: 'd', v: '2024-10-10T07:00:00.000Z', w: '10/10/24' }, + C3: { t: 's', v: '10:10:12' }, + D3: { t: 'n', w: '2', v: 2 }, + E3: { t: 'n', w: '0', v: 'good' }, + '!ref': 'A1:F3' + }; + + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const getDefaultWorksheetStub = sinon.stub(worksheetUtils, 'getDefaultWorksheet'); + const critterbaseInsertStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + + const critterAliasMap = new Map([ + [ + 'carl', + { + critter_id: 'A', + animal_id: 'carl', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }] + } as any + ], + [ + 'carlita', + { + critter_id: 'B', + animal_id: 'carlita', + itis_tsn: 'tsn2', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:12' }] + } as any + ] + ]); + + getDefaultWorksheetStub.returns(worksheet); + nonStandardColumnsStub.returns(['TAIL LENGTH', 'SKULL CONDITION']); + critterAliasMapStub.resolves(critterAliasMap); + critterbaseInsertStub.resolves({ + created: { qualitative_measurements: 1, quantitative_measurements: 1 } + } as IBulkCreateResponse); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { + qualitative: [ + { + taxon_measurement_id: 'Z', + measurement_name: 'skull condition', + options: [{ qualitative_option_id: 'C', option_label: 'good' }] + } + ], + quantitative: [ + { taxon_measurement_id: 'Z', measurement_name: 'tail length', min_value: 0, max_value: 10 } + ] + } as any + ], + [ + 'tsn2', + { + qualitative: [ + { + taxon_measurement_id: 'Z', + measurement_name: 'skull condition', + + options: [{ qualitative_option_id: 'C', option_label: 'good' }] + } + ], + quantitative: [ + { taxon_measurement_id: 'Z', measurement_name: 'tail length', min_value: 0, max_value: 10 } + ] + } as any + ] + ]) + ); + + try { + const data = await importCSV(new MediaFile('test', 'test', 'test' as unknown as Buffer), strategy); + expect(data).to.be.eql(2); + expect(critterbaseInsertStub).to.have.been.calledOnceWithExactly({ + qualitative_measurements: [ + { + critter_id: 'A', + capture_id: 'B', + taxon_measurement_id: 'Z', + qualitative_option_id: 'C' + }, + { + critter_id: 'B', + capture_id: 'B', + taxon_measurement_id: 'Z', + qualitative_option_id: 'C' + } + ], + quantitative_measurements: [ + { + critter_id: 'A', + capture_id: 'B', + taxon_measurement_id: 'Z', + value: 2 + }, + { + critter_id: 'B', + capture_id: 'B', + taxon_measurement_id: 'Z', + value: 2 + } + ] + }); + } catch (e: any) { + expect.fail(); + } + }); + }); + describe('_getTsnsMeasurementMap', () => { + it('should return correct taxon measurement mapping', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const getTaxonMeasurementsStub = sinon.stub( + strategy.surveyCritterService.critterbaseService, + 'getTaxonMeasurements' + ); + + const measurementA: any = { qualitative: [{ tsn: 'tsn1', measurement: 'measurement1' }], quantitative: [] }; + const measurementB: any = { quantitative: [{ tsn: 'tsn2', measurement: 'measurement2', qualitative: [] }] }; + + getTaxonMeasurementsStub.onCall(0).resolves(measurementA); + + getTaxonMeasurementsStub.onCall(1).resolves(measurementB); + + const tsns = ['tsn1', 'tsn2', 'tsn2']; + + const result = await strategy._getTsnsMeasurementMap(tsns); + + const expectedResult = new Map([ + ['tsn1', measurementA], + ['tsn2', measurementB] + ]); + + expect(result).to.be.deep.equal(expectedResult); + }); + }); + + describe('_getRowMeta', () => { + it('should return correct row meta', () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' }; + const critterAliasMap = new Map([ + [ + 'alias', + { + critter_id: 'A', + animal_id: 'alias', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + const result = strategy._getRowMeta(row, critterAliasMap); + + expect(result).to.be.deep.equal({ critter_id: 'A', tsn: 'tsn1', capture_id: 'B' }); + }); + + it('should return all undefined properties if unable to match critter', () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' }; + const critterAliasMap = new Map([ + [ + 'alias2', + { + critter_id: 'A', + animal_id: 'alias2', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + const result = strategy._getRowMeta(row, critterAliasMap); + + expect(result).to.be.deep.equal({ critter_id: undefined, tsn: undefined, capture_id: undefined }); + }); + + it('should undefined capture_id if unable to match timestamps', () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const row = { ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10' }; + const critterAliasMap = new Map([ + [ + 'alias', + { + critter_id: 'A', + animal_id: 'alias', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '11/11/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + const result = strategy._getRowMeta(row, critterAliasMap); + + expect(result).to.be.deep.equal({ critter_id: 'A', tsn: 'tsn1', capture_id: undefined }); + }); + }); + + describe('_validateQualitativeMeasurementCell', () => { + it('should return option_id when valid', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQualitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + measurement_desc: null, + options: [{ qualitative_option_id: 'C', option_label: 'measurement', option_value: 0, option_desc: 'desc' }] + }; + + const result = strategy._validateQualitativeMeasurementCell('measurement', measurement); + + expect(result.error).to.be.undefined; + expect(result.optionId).to.be.equal('C'); + }); + + it('should return error when invalid value', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQualitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + measurement_desc: null, + options: [{ qualitative_option_id: 'C', option_label: 'measurement', option_value: 0, option_desc: 'desc' }] + }; + + const result = strategy._validateQualitativeMeasurementCell('bad', measurement); + + expect(result.error).to.exist; + expect(result.optionId).to.be.undefined; + }); + }); + + describe('_validateQuantitativeMeasurementCell', () => { + it('should return value when valid', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQuantitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + unit: 'centimeter', + min_value: 0, + max_value: 10, + measurement_desc: null + }; + + const resultA = strategy._validateQuantitativeMeasurementCell(0, measurement); + + expect(resultA.error).to.be.undefined; + expect(resultA.value).to.be.equal(0); + + const resultB = strategy._validateQuantitativeMeasurementCell(10, measurement); + + expect(resultB.error).to.be.undefined; + expect(resultB.value).to.be.equal(10); + }); + + it('should return error when invalid value', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const measurement: CBQuantitativeMeasurementTypeDefinition = { + itis_tsn: 1, + taxon_measurement_id: 'A', + measurement_name: 'measurement', + unit: 'centimeter', + min_value: 0, + max_value: 10, + measurement_desc: null + }; + + const resultA = strategy._validateQuantitativeMeasurementCell(-1, measurement); + + expect(resultA.error).to.exist; + expect(resultA.value).to.be.undefined; + + const resultB = strategy._validateQuantitativeMeasurementCell(11, measurement); + + expect(resultB.error).to.exist; + expect(resultB.value).to.be.undefined; + }); + }); + + describe('_validateMeasurementCell', () => { + const critterAliasMap = new Map([ + [ + 'alias', + { + critter_id: 'A', + animal_id: 'alias', + itis_tsn: 'tsn1', + captures: [{ capture_id: 'B', capture_date: '10/10/2024', capture_time: '10:10:10' }] + } as any + ] + ]); + + it('should return no errors and data when valid rows', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'B' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect.fail(); + } else { + expect(result.data).to.be.deep.equal([ + { critter_id: 'A', capture_id: 'B', taxon_measurement_id: 'Z', qualitative_option_id: 'C' } + ]); + } + }); + + it('should return error when unable to map alias to critter', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: undefined, tsn: undefined, capture_id: undefined }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, message: 'Unable to find matching Critter with alias.' } + ]); + } else { + expect.fail(); + } + }); + + it('should return error when unable to map capture to critter', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: undefined }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, message: 'Unable to find matching Capture with date and time.' } + ]); + } else { + expect.fail(); + } + }); + + it('should return error when unable to map tsn to critter', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: undefined, capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: undefined, optionId: 'C' }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([{ row: 0, message: 'Unable to find ITIS TSN for Critter.' }]); + } else { + expect.fail(); + } + }); + + it('should return error when qualitative measurement validation fails', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQualitativeMeasurementCellStub = sinon.stub(strategy, '_validateQualitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { qualitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], quantitative: [] } as any + ] + ]) + ); + validateQualitativeMeasurementCellStub.returns({ error: 'qualitative failed', optionId: undefined }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([{ row: 0, col: 'MEASUREMENT', message: 'qualitative failed' }]); + } else { + expect.fail(); + } + }); + + it('should return error when quantitative measurement validation fails', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + const validateQuantitativeMeasurementCellStub = sinon.stub(strategy, '_validateQuantitativeMeasurementCell'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { quantitative: [{ taxon_measurement_id: 'Z', measurement_name: 'measurement' }], qualitative: [] } as any + ] + ]) + ); + validateQuantitativeMeasurementCellStub.returns({ error: 'quantitative failed', value: undefined }); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([{ row: 0, col: 'MEASUREMENT', message: 'quantitative failed' }]); + } else { + expect.fail(); + } + }); + + it('should return error when no measurements exist for taxon', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves(new Map([['tsn1', { quantitative: [], qualitative: [] } as any]])); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, col: 'MEASUREMENT', message: 'No measurements exist for this taxon.' } + ]); + } else { + expect.fail(); + } + }); + + it('should return error when no measurements exist for taxon', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const nonStandardColumnsStub = sinon.stub(strategy, '_getNonStandardColumns'); + const critterAliasMapStub = sinon.stub(strategy.surveyCritterService, 'getSurveyCritterAliasMap'); + const getRowMetaStub = sinon.stub(strategy, '_getRowMeta'); + const getTsnMeasurementMapStub = sinon.stub(strategy, '_getTsnsMeasurementMap'); + + nonStandardColumnsStub.returns(['MEASUREMENT']); + critterAliasMapStub.resolves(critterAliasMap); + getRowMetaStub.returns({ critter_id: 'A', tsn: 'tsn1', capture_id: 'C' }); + getTsnMeasurementMapStub.resolves( + new Map([ + [ + 'tsn1', + { quantitative: [{ measurement_name: 'notfound' }], qualitative: [{ measurement_name: 'notfound' }] } as any + ] + ]) + ); + + const rows = [{ ALIAS: 'alias', CAPTURE_DATE: '10/10/2024', CAPTURE_TIME: '10:10:10', measurement: 'length' }]; + + const result = await strategy.validateRows(rows, {}); + + if (!result.success) { + expect(result.error.issues).to.be.deep.equal([ + { row: 0, col: 'MEASUREMENT', message: 'Unable to match column name to an existing measurement.' } + ]); + } else { + expect.fail(); + } + }); + }); + describe('insert', () => { + it('should correctly format the insert payload for critterbase bulk insert', async () => { + const conn = getMockDBConnection(); + const strategy = new ImportMeasurementsStrategy(conn, 1); + + const bulkCreateStub = sinon.stub(strategy.surveyCritterService.critterbaseService, 'bulkCreate'); + + const rows = [ + { critter_id: 'A', capture_id: 'B', taxon_measurement_id: 'C', qualitative_option_id: 'D' }, + { critter_id: 'E', capture_id: 'F', taxon_measurement_id: 'G', value: 0 } + ]; + + bulkCreateStub.resolves({ + created: { qualitative_measurements: 1, quantitative_measurements: 1 } + } as IBulkCreateResponse); + + const result = await strategy.insert(rows); + + expect(bulkCreateStub).to.have.been.calledOnceWithExactly({ + qualitative_measurements: [ + { critter_id: 'A', capture_id: 'B', taxon_measurement_id: 'C', qualitative_option_id: 'D' } + ], + quantitative_measurements: [{ critter_id: 'E', capture_id: 'F', taxon_measurement_id: 'G', value: 0 }] + }); + expect(result).to.be.eql(2); + }); + }); +}); diff --git a/api/src/services/import-services/measurement/import-measurements-strategy.ts b/api/src/services/import-services/measurement/import-measurements-strategy.ts new file mode 100644 index 0000000000..d39e69f86c --- /dev/null +++ b/api/src/services/import-services/measurement/import-measurements-strategy.ts @@ -0,0 +1,344 @@ +import { uniq } from 'lodash'; +import { WorkSheet } from 'xlsx'; +import { IDBConnection } from '../../../database/db'; +import { getLogger } from '../../../utils/logger'; +import { CSV_COLUMN_ALIASES } from '../../../utils/xlsx-utils/column-aliases'; +import { generateColumnCellGetterFromColumnValidator } from '../../../utils/xlsx-utils/column-validator-utils'; +import { getNonStandardColumnNamesFromWorksheet, IXLSXCSVValidator } from '../../../utils/xlsx-utils/worksheet-utils'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition, + ICritterDetailed +} from '../../critterbase-service'; +import { DBService } from '../../db-service'; +import { SurveyCritterService } from '../../survey-critter-service'; +import { CSVImportStrategy, Row, Validation, ValidationError } from '../import-csv.interface'; +import { findCapturesFromDateTime } from '../utils/datetime'; +import { + CsvMeasurement, + CsvQualitativeMeasurement, + CsvQuantitativeMeasurement +} from './import-measurements-strategy.interface'; + +const defaultLog = getLogger('services/import/import-measurements-strategy'); + +/** + * + * ImportMeasurementsStrategy - Injected into importCSV as the CSV import dependency + * + * @example new CSVImport(new ImportMeasurementsStrategy(connection, surveyId)).import(file); + * + * @class ImportMeasurementsStrategy + * @extends DBService + * + */ +export class ImportMeasurementsStrategy extends DBService implements CSVImportStrategy { + surveyCritterService: SurveyCritterService; + + surveyId: number; + + /** + * An XLSX validation config for the standard columns of a Measurement CSV. + * + * Note: `satisfies` allows `keyof` to correctly infer key types, while also + * enforcing uppercase object keys. + */ + columnValidator = { + ALIAS: { type: 'string', aliases: CSV_COLUMN_ALIASES.ALIAS }, + CAPTURE_DATE: { type: 'date' }, + CAPTURE_TIME: { type: 'string', optional: true } + } satisfies IXLSXCSVValidator; + + /** + * Instantiates an instance of ImportMeasurementsStrategy + * + * @param {IDBConnection} connection - Database connection + * @param {number} surveyId - Survey identifier + */ + constructor(connection: IDBConnection, surveyId: number) { + super(connection); + + this.surveyId = surveyId; + + this.surveyCritterService = new SurveyCritterService(connection); + } + + /** + * Get non-standard columns (measurement columns) from worksheet. + * + * @param {WorkSheet} worksheet - Xlsx worksheet + * @returns {string[]} Array of non-standard headers from CSV (worksheet) + */ + _getNonStandardColumns(worksheet: WorkSheet) { + return uniq(getNonStandardColumnNamesFromWorksheet(worksheet, this.columnValidator)); + } + + /** + * Get TSN measurement map for validation. + * + * For a list of TSNS return all measurements inherited or directly assigned. + * + * @async + * @param {string[]} tsns - List of ITIS TSN's + * @returns {*} + */ + async _getTsnsMeasurementMap(tsns: string[]) { + const tsnMeasurementMap = new Map< + string, + { + qualitative: CBQualitativeMeasurementTypeDefinition[]; + quantitative: CBQuantitativeMeasurementTypeDefinition[]; + } + >(); + + const uniqueTsns = [...new Set(tsns)]; + + const measurements = await Promise.all( + uniqueTsns.map((tsn) => this.surveyCritterService.critterbaseService.getTaxonMeasurements(tsn)) + ); + + uniqueTsns.forEach((tsn, index) => { + tsnMeasurementMap.set(tsn, measurements[index]); + }); + + return tsnMeasurementMap; + } + + /** + * Get row meta data for validation. + * + * @param {Row} row - CSV row + * @param {Map} critterAliasMap - Survey critter alias mapping + * @returns {{ capture_id?: string; critter_id?: string; tsn?: string }} + */ + _getRowMeta( + row: Row, + critterAliasMap: Map + ): { capture_id?: string; critter_id?: string; tsn?: string } { + const getColumnCell = generateColumnCellGetterFromColumnValidator(this.columnValidator); + + const alias = getColumnCell(row, 'ALIAS'); + const captureDate = getColumnCell(row, 'CAPTURE_DATE'); + const captureTime = getColumnCell(row, 'CAPTURE_TIME'); + + let capture_id, critter_id, tsn; + + if (alias.cell) { + const critter = critterAliasMap.get(alias.cell.toLowerCase()); + + if (critter) { + const captures = findCapturesFromDateTime(critter.captures, captureDate.cell, captureTime.cell); + critter_id = critter.critter_id; + capture_id = captures.length === 1 ? captures[0].capture_id : undefined; + tsn = String(critter.itis_tsn); // Cast to string for convienience + } + } + + return { critter_id, capture_id, tsn }; + } + + /** + * Validate qualitative measurement. + * + * @param {string} cell - CSV measurement cell value + * @param {CBQualitativeMeasurementTypeDefinition} measurement - Found qualitative measurement match + * @returns {*} + */ + _validateQualitativeMeasurementCell(cell: string, measurement: CBQualitativeMeasurementTypeDefinition) { + if (typeof cell !== 'string') { + return { error: 'Qualitative measurement expecting text value.', optionId: undefined }; + } + + const matchingOptionValue = measurement.options.find( + (option) => option.option_label.toLowerCase() === cell.toLowerCase() + ); + + // Validate cell value is an alowed qualitative measurement option + if (!matchingOptionValue) { + return { + error: `Incorrect qualitative measurement value. Allowed: ${measurement.options.map((option) => + option.option_label.toLowerCase() + )}`, + optionId: undefined + }; + } + + return { error: undefined, optionId: matchingOptionValue.qualitative_option_id }; + } + + /** + * Validate quantitative measurement + * + * @param {number} cell - CSV measurement cell value + * @param {CBQuantitativeMeasurementTypeDefinition} measurement - Found quantitative measurement match + * @returns {*} + */ + _validateQuantitativeMeasurementCell(cell: number, measurement: CBQuantitativeMeasurementTypeDefinition) { + if (typeof cell !== 'number') { + return { error: 'Quantitative measurement expecting number value.', value: undefined }; + } + + // Validate cell value is withing the measurement min max bounds + if (measurement.max_value != null && cell > measurement.max_value) { + return { error: 'Quantitative measurement out of bounds. Too small.', value: undefined }; + } + + if (measurement.min_value != null && cell < measurement.min_value) { + return { error: 'Quantitative measurement out of bounds. Too small.' }; + } + + return { error: undefined, value: cell }; + } + + /** + * Validate CSV worksheet rows against reference data. + * + * Note: This function is longer than I would like, but moving logic into seperate methods + * made the flow more complex and equally as long. + * + * @async + * @param {Row[]} rows - Invalidated CSV rows + * @param {WorkSheet} worksheet - Xlsx worksheet + * @returns {*} + */ + async validateRows(rows: Row[], worksheet: WorkSheet): Promise> { + // Generate type-safe cell getter from column validator + const nonStandardColumns = this._getNonStandardColumns(worksheet); + + // Get Critterbase reference data + const critterAliasMap = await this.surveyCritterService.getSurveyCritterAliasMap(this.surveyId); + const rowTsns = rows.map((row) => this._getRowMeta(row, critterAliasMap).tsn).filter(Boolean) as string[]; + const tsnMeasurementsMap = await this._getTsnsMeasurementMap(rowTsns); + + const rowErrors: ValidationError[] = []; + const validatedRows: CsvMeasurement[] = []; + + rows.forEach((row, index) => { + const { critter_id, capture_id, tsn } = this._getRowMeta(row, critterAliasMap); + + // Validate critter can be matched via alias + if (!critter_id) { + rowErrors.push({ row: index, message: 'Unable to find matching Critter with alias.' }); + return; + } + + // Validate capture can be matched with date and time + if (!capture_id) { + rowErrors.push({ row: index, message: 'Unable to find matching Capture with date and time.' }); + return; + } + + // This will only be triggered with an invalid alias + if (!tsn) { + rowErrors.push({ row: index, message: 'Unable to find ITIS TSN for Critter.' }); + return; + } + + // Loop through all non-standard (measurement) columns + for (const column of nonStandardColumns) { + // Get the cell value from the row (case insensitive) + const cellValue = row[column] ?? row[column.toLowerCase()] ?? row[column.toUpperCase()]; + + // If the cell value is null or undefined - skip validation + if (cellValue == null) { + continue; + } + + const measurements = tsnMeasurementsMap.get(tsn); + + // Validate taxon has reference measurements in Critterbase + if (!measurements || (!measurements.quantitative.length && !measurements.qualitative.length)) { + rowErrors.push({ row: index, col: column, message: 'No measurements exist for this taxon.' }); + continue; + } + + const qualitativeMeasurement = measurements?.qualitative.find( + (measurement) => measurement.measurement_name.toLowerCase() === column.toLowerCase() + ); + + // QUALITATIVE MEASUREMENT VALIDATION + if (qualitativeMeasurement) { + const { error, optionId } = this._validateQualitativeMeasurementCell(cellValue, qualitativeMeasurement); + + if (error !== undefined) { + rowErrors.push({ row: index, col: column, message: error }); + } else { + // Assign qualitative measurement to validated rows + validatedRows.push({ + critter_id, + capture_id, + taxon_measurement_id: qualitativeMeasurement.taxon_measurement_id, + qualitative_option_id: optionId + }); + } + + continue; + } + + const quantitativeMeasurement = measurements?.quantitative.find( + (measurement) => measurement.measurement_name.toLowerCase() === column.toLowerCase() + ); + + // QUANTITATIVE MEASUREMENT VALIDATION + if (quantitativeMeasurement) { + const { error, value } = this._validateQuantitativeMeasurementCell(cellValue, quantitativeMeasurement); + + if (error !== undefined) { + rowErrors.push({ row: index, col: column, message: error }); + } else { + // Assign quantitative measurement to validated rows + validatedRows.push({ + critter_id, + capture_id, + taxon_measurement_id: quantitativeMeasurement.taxon_measurement_id, + value: value + }); + } + + continue; + } + + // Validate the column header is a known Critterbase measurement + rowErrors.push({ + row: index, + col: column, + message: 'Unable to match column name to an existing measurement.' + }); + } + }); + + if (!rowErrors.length) { + return { success: true, data: validatedRows }; + } + + return { success: false, error: { issues: rowErrors } }; + } + + /** + * Insert CSV measurements into Critterbase. + * + * @async + * @param {CsvCritter[]} measurements - CSV row measurements + * @returns {Promise} List of inserted measurements + */ + async insert(measurements: CsvMeasurement[]): Promise { + const qualitative_measurements = measurements.filter( + (measurement): measurement is CsvQualitativeMeasurement => 'qualitative_option_id' in measurement + ); + + const quantitative_measurements = measurements.filter( + (measurement): measurement is CsvQuantitativeMeasurement => 'value' in measurement + ); + + const response = await this.surveyCritterService.critterbaseService.bulkCreate({ + qualitative_measurements, + quantitative_measurements + }); + + const measurementCount = response.created.qualitative_measurements + response.created.quantitative_measurements; + + defaultLog.debug({ label: 'import measurements', measurements, insertedCount: measurementCount }); + + return measurementCount; + } +} diff --git a/api/src/services/observation-service.ts b/api/src/services/observation-service.ts index b5ae70c09d..2d5d48f22d 100644 --- a/api/src/services/observation-service.ts +++ b/api/src/services/observation-service.ts @@ -43,7 +43,7 @@ import { validateMeasurements } from '../utils/observation-xlsx-utils/measurement-column-utils'; import { CSV_COLUMN_ALIASES } from '../utils/xlsx-utils/column-aliases'; -import { generateCellGetterFromColumnValidator } from '../utils/xlsx-utils/column-validator-utils'; +import { generateColumnCellGetterFromColumnValidator } from '../utils/xlsx-utils/column-validator-utils'; import { constructXLSXWorkbook, getDefaultWorksheet, @@ -82,7 +82,7 @@ export const observationStandardColumnValidator = { LONGITUDE: { type: 'number', aliases: CSV_COLUMN_ALIASES.LONGITUDE } } satisfies IXLSXCSVValidator; -export const getCellValue = generateCellGetterFromColumnValidator(observationStandardColumnValidator); +export const getColumnCellValue = generateColumnCellGetterFromColumnValidator(observationStandardColumnValidator); export interface InsertSubCount { observation_subcount_id: number | null; @@ -590,7 +590,7 @@ export class ObservationService extends DBService { const newRowData: InsertUpdateObservations[] = worksheetRowObjects.map((row) => { const newSubcount: InsertSubCount = { observation_subcount_id: null, - subcount: getCellValue(row, 'COUNT') as number, + subcount: getColumnCellValue(row, 'COUNT').cell as number, qualitative_measurements: [], quantitative_measurements: [], qualitative_environments: [], @@ -616,16 +616,16 @@ export class ObservationService extends DBService { return { standardColumns: { survey_id: surveyId, - itis_tsn: getCellValue(row, 'ITIS_TSN') as number, + itis_tsn: getColumnCellValue(row, 'ITIS_TSN').cell as number, itis_scientific_name: null, survey_sample_site_id: samplePeriodHierarchyIds?.survey_sample_site_id ?? null, survey_sample_method_id: samplePeriodHierarchyIds?.survey_sample_method_id ?? null, survey_sample_period_id: samplePeriodHierarchyIds?.survey_sample_period_id ?? null, - latitude: getCellValue(row, 'LATITUDE') as number, - longitude: getCellValue(row, 'LONGITUDE') as number, - count: getCellValue(row, 'COUNT') as number, - observation_time: getCellValue(row, 'TIME') as string, - observation_date: getCellValue(row, 'DATE') as string + latitude: getColumnCellValue(row, 'LATITUDE').cell as number, + longitude: getColumnCellValue(row, 'LONGITUDE').cell as number, + count: getColumnCellValue(row, 'COUNT').cell as number, + observation_time: getColumnCellValue(row, 'TIME').cell as string, + observation_date: getColumnCellValue(row, 'DATE').cell as string }, subcounts: [newSubcount] }; @@ -671,7 +671,7 @@ export class ObservationService extends DBService { } const measurement = getMeasurementFromTsnMeasurementTypeDefinitionMap( - getCellValue(row, 'ITIS_TSN') as string, + getColumnCellValue(row, 'ITIS_TSN').cell as string, mColumn, tsnMeasurements ); diff --git a/api/src/utils/xlsx-utils/column-validator-utils.test.ts b/api/src/utils/xlsx-utils/column-validator-utils.test.ts index 243c7fe8e3..46ff4044b0 100644 --- a/api/src/utils/xlsx-utils/column-validator-utils.test.ts +++ b/api/src/utils/xlsx-utils/column-validator-utils.test.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; import { - generateCellGetterFromColumnValidator, + generateColumnCellGetterFromColumnValidator, getColumnAliasesFromValidator, getColumnNamesFromValidator } from './column-validator-utils'; @@ -26,19 +26,27 @@ describe('column-validator-utils', () => { }); }); - describe('generateCellGetterFromColumnValidator', () => { - const getCellValue = generateCellGetterFromColumnValidator(columnValidator); + describe('generateColumnCellGetterFromColumnValidator', () => { + const getCellValue = generateColumnCellGetterFromColumnValidator(columnValidator); it('should return the cell value for a known column name', () => { - expect(getCellValue({ NAME: 'Dr. Steve Brule' }, 'NAME')).to.be.eql('Dr. Steve Brule'); + expect(getCellValue({ NAME: 'Dr. Steve Brule' }, 'NAME').cell).to.be.eql('Dr. Steve Brule'); }); it('should return the cell value for a known alias name', () => { - expect(getCellValue({ IDENTIFIER: 1 }, 'ID')).to.be.eql(1); + expect(getCellValue({ IDENTIFIER: 1 }, 'ID').cell).to.be.eql(1); }); it('should return undefined if row cannot find cell value', () => { - expect(getCellValue({ BAD_NAME: 1 }, 'NAME')).to.be.eql(undefined); + expect(getCellValue({ BAD_NAME: 1 }, 'NAME').cell).to.be.eql(undefined); + }); + + it('should return column name', () => { + expect(getCellValue({ NAME: 1 }, 'NAME').column).to.be.eql('NAME'); + }); + + it('should return column alias name', () => { + expect(getCellValue({ IDENTIFIER: 1 }, 'ID').column).to.be.eql('IDENTIFIER'); }); }); }); diff --git a/api/src/utils/xlsx-utils/column-validator-utils.ts b/api/src/utils/xlsx-utils/column-validator-utils.ts index b570cb321d..c1f4ff5384 100644 --- a/api/src/utils/xlsx-utils/column-validator-utils.ts +++ b/api/src/utils/xlsx-utils/column-validator-utils.ts @@ -51,29 +51,29 @@ export const getColumnValidatorSpecification = (columnValidator: IXLSXCSVValidat }; /** - * Generate a cell getter from a column validator. + * Generate a column + cell getter from a column validator. * - * Note: This will attempt to retrive the cell value from the row by the known header first. + * Note: This will attempt to retrive the column header and cell value from the row by the known header first. * If not found, it will then attempt to retrieve the value by the column header aliases. * - * TODO: Can the internal typing for this be improved (without the `as` cast)? - * * @example - * const getCellValue = generateCellGetterFromColumnValidator(columnValidator) - * const itis_tsn = getCellValue(row, 'ITIS_TSN') + * const getColumnCell = generateColumnCellGetterFromColumnValidator(columnValidator) + * + * const itis_tsn = getColumnCell(row, 'ITIS_TSN').cell + * const tsnColumn = getColumnCell(row, 'ITIS_TSN').column * * @template T * @param {T} columnValidator - Column validator * @returns {*} */ -export const generateCellGetterFromColumnValidator = (columnValidator: T) => { - return (row: Row, validatorKey: keyof T): J | undefined => { +export const generateColumnCellGetterFromColumnValidator = (columnValidator: T) => { + return (row: Row, validatorKey: keyof T): { column: string; cell: J | undefined } => { // Cast the columnValidatorKey to a string for convienience const key = validatorKey as string; - // Attempt to retrieve the cell value from the default column name + // Attempt to retrieve the column and cell value from the default column name if (row[key]) { - return row[key]; + return { column: key, cell: row[key] }; } const columnSpec = columnValidator[validatorKey] as IXLSXCSVColumn; @@ -81,11 +81,14 @@ export const generateCellGetterFromColumnValidator = { const worksheetRows = getWorksheetRowObjects(worksheet); const columnNames = getColumnNamesFromValidator(columnValidator); - const getCellValue = generateCellGetterFromColumnValidator(columnValidator); + const getCellValue = generateColumnCellGetterFromColumnValidator(columnValidator); return worksheetRows.every((row) => { return columnNames.every((columnName, index) => { - const value = getCellValue(row, columnName.toUpperCase() as Uppercase); + const value = getCellValue(row, columnName.toUpperCase() as Uppercase).cell; const type = typeof value; const columnSpec: IXLSXCSVColumn = columnValidator[columnName]; From 7eb4c5cf43c1212c1d0fccdcc475116a0125931c Mon Sep 17 00:00:00 2001 From: Macgregor Aubertin-Young <108430771+mauberti-bc@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:29:43 -0700 Subject: [PATCH 3/3] Observation Analytics (#1329) Add observation analytics Feature: UI, Endpoint/Service/Repo --------- Co-authored-by: Nick Phura --- api/package-lock.json | 2 +- api/package.json | 2 +- api/src/models/observation-analytics.ts | 76 +++ api/src/paths/analytics/observations.ts | 243 ++++++++ .../observations/measurements/index.ts | 134 +++++ api/src/paths/telemetry/deployments.ts | 2 - .../repositories/analytics-repository.test.ts | 76 +++ api/src/repositories/analytics-repository.ts | 102 ++++ api/src/services/analytics-service.test.ts | 518 ++++++++++++++++++ api/src/services/analytics-service.ts | 282 ++++++++++ app/src/features/surveys/view/SurveyPage.tsx | 8 +- .../analytics/SurveyObservationAnalytics.tsx | 194 +++++++ .../ObservationAnalyticsDataTable.tsx | 68 +++ ...ObservationAnalyticsDataTableContainer.tsx | 154 ++++++ .../ObservationAnalyticsNoDataOverlay.tsx | 49 ++ ...rvationsAnalyticsGridColumnDefinitions.tsx | 221 ++++++++ .../SurveyObservationTabularDataContainer.tsx | 81 +++ .../observation/SurveySpatialObservation.tsx | 8 +- .../SurveySpatialObservationTable.tsx | 5 +- app/src/hooks/api/useAnalyticsApi.test.ts | 65 +++ app/src/hooks/api/useAnalyticsApi.ts | 36 ++ app/src/hooks/api/useObservationApi.ts | 27 + app/src/hooks/useBioHubApi.ts | 4 + .../interfaces/useAnalyticsApi.interface.ts | 31 ++ .../useSamplingSiteApi.interface.ts | 1 + 25 files changed, 2376 insertions(+), 13 deletions(-) create mode 100644 api/src/models/observation-analytics.ts create mode 100644 api/src/paths/analytics/observations.ts create mode 100644 api/src/paths/project/{projectId}/survey/{surveyId}/observations/measurements/index.ts create mode 100644 api/src/repositories/analytics-repository.test.ts create mode 100644 api/src/repositories/analytics-repository.ts create mode 100644 api/src/services/analytics-service.test.ts create mode 100644 api/src/services/analytics-service.ts create mode 100644 app/src/features/surveys/view/components/analytics/SurveyObservationAnalytics.tsx create mode 100644 app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTable.tsx create mode 100644 app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer.tsx create mode 100644 app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsNoDataOverlay.tsx create mode 100644 app/src/features/surveys/view/components/analytics/components/ObservationsAnalyticsGridColumnDefinitions.tsx create mode 100644 app/src/features/surveys/view/components/data-container/SurveyObservationTabularDataContainer.tsx create mode 100644 app/src/hooks/api/useAnalyticsApi.test.ts create mode 100644 app/src/hooks/api/useAnalyticsApi.ts create mode 100644 app/src/interfaces/useAnalyticsApi.interface.ts diff --git a/api/package-lock.json b/api/package-lock.json index aae5dc84af..6ac5daca69 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -47,7 +47,7 @@ "winston": "^3.3.3", "winston-daily-rotate-file": "^5.0.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz", - "zod": "^3.23.0" + "zod": "^3.23.8" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.1", diff --git a/api/package.json b/api/package.json index 620cb214f7..fe2fd2dcb2 100644 --- a/api/package.json +++ b/api/package.json @@ -64,7 +64,7 @@ "winston": "^3.3.3", "winston-daily-rotate-file": "^5.0.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz", - "zod": "^3.23.0" + "zod": "^3.23.8" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.1", diff --git a/api/src/models/observation-analytics.ts b/api/src/models/observation-analytics.ts new file mode 100644 index 0000000000..faff4a53f8 --- /dev/null +++ b/api/src/models/observation-analytics.ts @@ -0,0 +1,76 @@ +import { z } from 'zod'; + +export const QualitativeMeasurementAnalyticsSchema = z.object({ + option: z.object({ + option_id: z.string(), + option_label: z.string() + }), + taxon_measurement_id: z.string(), + measurement_name: z.string() +}); + +export type QualitativeMeasurementAnalytics = z.infer; + +export const QuantitativeMeasurementAnalyticsSchema = z.object({ + value: z.number(), + taxon_measurement_id: z.string(), + measurement_name: z.string() +}); + +export type QuantitativeMeasurementAnalytics = z.infer; + +export const ObservationCountByGroupSchema = z.object({ + row_count: z.number(), + individual_count: z.number(), + individual_percentage: z.number() +}); + +export type ObservationCountByGroup = z.infer; + +export const ObservationCountByGroupWithNamedMeasurementsSchema = ObservationCountByGroupSchema.extend({ + qualitative_measurements: z.array(QualitativeMeasurementAnalyticsSchema), + quantitative_measurements: z.array(QuantitativeMeasurementAnalyticsSchema) +}); + +export type ObservationCountByGroupWithNamedMeasurements = z.infer< + typeof ObservationCountByGroupWithNamedMeasurementsSchema +>; + +export const ObservationCountByGroupWithMeasurementsSchema = z.object({ + quant_measurements: z.array( + z.object({ + value: z.number().nullable(), + critterbase_taxon_measurement_id: z.string() + }) + ), + qual_measurements: z.array( + z.object({ + option_id: z.string().nullable(), + critterbase_taxon_measurement_id: z.string() + }) + ) +}); + +export type ObservationCountByGroupWithMeasurements = z.infer; + +export const ObservationCountByGroupSQLResponse = z + .object({ + id: z.string(), + row_count: z.number(), + individual_count: z.number(), + individual_percentage: z.number(), + quant_measurements: z.record(z.string(), z.number().nullable()), + qual_measurements: z.record(z.string(), z.string().nullable()) + }) + // Allow additional properties + .catchall(z.any()); + +export type ObservationCountByGroupSQLResponse = z.infer; + +export const ObservationAnalyticsResponse = ObservationCountByGroupWithNamedMeasurementsSchema.merge( + ObservationCountByGroupSchema +) + // Allow additional properties + .catchall(z.any()); + +export type ObservationAnalyticsResponse = z.infer; diff --git a/api/src/paths/analytics/observations.ts b/api/src/paths/analytics/observations.ts new file mode 100644 index 0000000000..65cbc6265e --- /dev/null +++ b/api/src/paths/analytics/observations.ts @@ -0,0 +1,243 @@ +import { Operation } from 'express-openapi'; +import { RequestHandler } from 'http-proxy-middleware'; +import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../constants/roles'; +import { getDBConnection } from '../../database/db'; +import { authorizeRequestHandler } from '../../request-handlers/security/authorization'; +import { AnalyticsService } from '../../services/analytics-service'; +import { getLogger } from '../../utils/logger'; + +const defaultLog = getLogger('paths/analytics/observations'); + +export const GET: Operation = [ + authorizeRequestHandler((req) => { + return { + or: [ + { + validProjectPermissions: [ + PROJECT_PERMISSION.COORDINATOR, + PROJECT_PERMISSION.COLLABORATOR, + PROJECT_PERMISSION.OBSERVER + ], + surveyId: Number(req.params.surveyId), + discriminator: 'ProjectPermission' + }, + { + validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + }, + { + validSystemRoles: [SYSTEM_ROLE.SYSTEM_ADMIN], + discriminator: 'SystemRole' + } + ] + }; + }), + getObservationCountByGroup() +]; + +GET.apiDoc = { + description: 'get analytics about observations for one or more surveys', + tags: ['analytics'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + in: 'query', + name: 'surveyIds', + schema: { + type: 'array', + items: { + type: 'integer', + minimum: 1 + } + }, + required: true + }, + { + in: 'query', + name: 'groupByColumns', + schema: { + type: 'array', + items: { + type: 'string' + } + }, + description: 'An array of column names to group the observations data by' + }, + { + in: 'query', + name: 'groupByQuantitativeMeasurements', + schema: { + type: 'array', + items: { + type: 'string' + } + }, + description: 'An array of quantitative taxon_measurement_ids to group the observations data by' + }, + { + in: 'query', + name: 'groupByQualitativeMeasurements', + schema: { + type: 'array', + items: { + type: 'string' + } + }, + description: 'An array of qualitative taxon_measurement_ids to group the observations data by' + } + ], + responses: { + 200: { + description: 'Analytics calculated OK.', + content: { + 'application/json': { + schema: { + title: 'Observation analytics response object', + type: 'array', + items: { + type: 'object', + required: [ + 'id', + 'row_count', + 'individual_count', + 'individual_percentage', + 'quantitative_measurements', + 'qualitative_measurements' + ], + // Additional properties is intentionally true to allow for dynamic key-value measurement pairs + additionalProperties: true, + properties: { + id: { + type: 'string', + format: 'uuid', + description: 'Unique identifier for the group. Will not be consistent between requests.' + }, + row_count: { + type: 'number', + description: 'Number of rows in the group' + }, + individual_count: { + type: 'number', + description: 'Sum of subcount values across all rows in the group' + }, + individual_percentage: { + type: 'number', + description: + 'Sum of subcount values across the group divided by the sum of subcount values across all observations in the specified surveys' + }, + quantitative_measurements: { + type: 'array', + items: { + type: 'object', + description: 'Quantitative measurement groupings', + required: ['taxon_measurement_id', 'measurement_name', 'value'], + additionalProperties: false, + properties: { + taxon_measurement_id: { + type: 'string', + format: 'uuid' + }, + measurement_name: { + type: 'string' + }, + value: { + type: 'number', + nullable: true + } + } + } + }, + qualitative_measurements: { + type: 'array', + items: { + type: 'object', + description: 'Qualitative measurement groupings', + required: ['taxon_measurement_id', 'measurement_name', 'option'], + additionalProperties: false, + properties: { + taxon_measurement_id: { + type: 'string', + format: 'uuid' + }, + measurement_name: { + type: 'string' + }, + option: { + type: 'object', + required: ['option_id', 'option_label'], + additionalProperties: false, + properties: { + option_id: { + type: 'string', + format: 'uuid', + nullable: true + }, + option_label: { + type: 'string', + nullable: true + } + } + } + } + } + } + } + } + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 403: { + $ref: '#/components/responses/403' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +export function getObservationCountByGroup(): RequestHandler { + return async (req, res) => { + defaultLog.debug({ label: 'getObservationCountByGroup' }); + + const connection = getDBConnection(req.keycloak_token); + + try { + const { surveyIds, groupByColumns, groupByQuantitativeMeasurements, groupByQualitativeMeasurements } = req.query; + + await connection.open(); + + const analyticsService = new AnalyticsService(connection); + + const response = await analyticsService.getObservationCountByGroup( + (surveyIds as string[]).map(Number), + (groupByColumns as string[]) ?? [], + (groupByQuantitativeMeasurements as string[]) ?? [], + (groupByQualitativeMeasurements as string[]) ?? [] + ); + + await connection.commit(); + + return res.status(200).json(response); + } catch (error) { + defaultLog.error({ label: 'getObservationCountByGroup', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/observations/measurements/index.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/measurements/index.ts new file mode 100644 index 0000000000..c0e885ce6b --- /dev/null +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/observations/measurements/index.ts @@ -0,0 +1,134 @@ +import { Operation } from 'express-openapi'; +import { RequestHandler } from 'http-proxy-middleware'; +import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles'; +import { getDBConnection } from '../../../../../../../database/db'; +import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization'; +import { SubCountService } from '../../../../../../../services/subcount-service'; +import { getLogger } from '../../../../../../../utils/logger'; + +const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/observations/measurements'); + +export const GET: Operation = [ + authorizeRequestHandler((req) => { + return { + or: [ + { + validProjectPermissions: [ + PROJECT_PERMISSION.COORDINATOR, + PROJECT_PERMISSION.COLLABORATOR, + PROJECT_PERMISSION.OBSERVER + ], + surveyId: Number(req.params.surveyId), + discriminator: 'ProjectPermission' + }, + { + validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR], + discriminator: 'SystemRole' + } + ] + }; + }), + getSurveyObservationMeasurements() +]; + +GET.apiDoc = { + description: 'Get all measurement definitions for the survey.', + tags: ['observation'], + security: [ + { + Bearer: [] + } + ], + parameters: [ + { + in: 'path', + name: 'projectId', + schema: { + type: 'integer', + minimum: 1 + }, + required: true + }, + { + in: 'path', + name: 'surveyId', + schema: { + type: 'integer', + minimum: 1 + }, + required: true + } + ], + responses: { + 200: { + description: 'Observation measurements response', + content: { + 'application/json': { + schema: { + description: 'Qualitative and quantitative observation definitions for the survey', + type: 'object', + additionalProperties: false, + required: ['qualitative_measurements', 'quantitative_measurements'], + properties: { + qualitative_measurements: { + type: 'array', + items: {} + }, + quantitative_measurements: { + type: 'array', + items: {} + } + } + } + } + } + }, + 400: { + $ref: '#/components/responses/400' + }, + 401: { + $ref: '#/components/responses/401' + }, + 403: { + $ref: '#/components/responses/403' + }, + 500: { + $ref: '#/components/responses/500' + }, + default: { + $ref: '#/components/responses/default' + } + } +}; + +/** + * Fetch definitions of all measured for a given survey. + * + * @export + * @return {*} {RequestHandler} + */ +export function getSurveyObservationMeasurements(): RequestHandler { + return async (req, res) => { + const surveyId = Number(req.params.surveyId); + + defaultLog.debug({ label: 'getSurveyObservationMeasurement', surveyId }); + + const connection = getDBConnection(req.keycloak_token); + + try { + await connection.open(); + + const subcountService = new SubCountService(connection); + + const observationData = await subcountService.getMeasurementTypeDefinitionsForSurvey(surveyId); + + return res.status(200).json(observationData); + } catch (error) { + defaultLog.error({ label: 'getSurveyObservationMeasurements', message: 'error', error }); + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + }; +} diff --git a/api/src/paths/telemetry/deployments.ts b/api/src/paths/telemetry/deployments.ts index cc7b0b0d5d..035276ad86 100644 --- a/api/src/paths/telemetry/deployments.ts +++ b/api/src/paths/telemetry/deployments.ts @@ -37,8 +37,6 @@ GET.apiDoc = { type: 'array', items: { type: 'string', format: 'uuid', minimum: 1 } }, - explode: false, - style: 'form', required: true } ], diff --git a/api/src/repositories/analytics-repository.test.ts b/api/src/repositories/analytics-repository.test.ts new file mode 100644 index 0000000000..66f3decda4 --- /dev/null +++ b/api/src/repositories/analytics-repository.test.ts @@ -0,0 +1,76 @@ +import chai, { expect } from 'chai'; +import { QueryResult } from 'pg'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { getMockDBConnection } from '../__mocks__/db'; +import { AnalyticsRepository } from './analytics-repository'; + +chai.use(sinonChai); + +describe('AnalyticsRepository', () => { + it('should construct', () => { + const mockDBConnection = getMockDBConnection(); + const analyticsRepository = new AnalyticsRepository(mockDBConnection); + + expect(analyticsRepository).to.be.instanceof(AnalyticsRepository); + }); + + describe('getObservationCountByGroup', () => { + it('Creates and executes sql query with empty params', async () => { + const mockRows = [ + { + row_count: 10, + individual_count: 5, + individual_percentage: 50, + quant_measurements: {}, + qual_measurements: { + critterbase_taxon_measurement_id: '1', + option_id: '2' + } + } + ]; + const mockQueryResponse = { rows: mockRows, rowCount: 1 } as unknown as QueryResult; + + const mockDBConnection = getMockDBConnection({ + knex: sinon.stub().resolves(mockQueryResponse) + }); + + const analyticsRepository = new AnalyticsRepository(mockDBConnection); + + const response = await analyticsRepository.getObservationCountByGroup([], [], [], []); + + expect(response).to.be.an('array'); + }); + + it('Creates and executes sql query with non-empty params', async () => { + const mockRows = [ + { + row_count: 10, + individual_count: 5, + individual_percentage: 50, + quant_measurements: {}, + qual_measurements: { + critterbase_taxon_measurement_id: '1', + option_id: '2' + } + } + ]; + const mockQueryResponse = { rows: mockRows, rowCount: 1 } as unknown as QueryResult; + + const mockDBConnection = getMockDBConnection({ + knex: sinon.stub().resolves(mockQueryResponse) + }); + + const analyticsRepository = new AnalyticsRepository(mockDBConnection); + + const response = await analyticsRepository.getObservationCountByGroup( + [1, 2, 3], + ['column1', 'column2'], + ['quant1', 'quant2'], + ['qual1', 'qual2'] + ); + + expect(response).to.be.an('array'); + }); + }); +}); diff --git a/api/src/repositories/analytics-repository.ts b/api/src/repositories/analytics-repository.ts new file mode 100644 index 0000000000..c9ebb232ba --- /dev/null +++ b/api/src/repositories/analytics-repository.ts @@ -0,0 +1,102 @@ +import { getKnex } from '../database/db'; +import { ObservationCountByGroupSQLResponse } from '../models/observation-analytics'; +import { BaseRepository } from './base-repository'; + +export class AnalyticsRepository extends BaseRepository { + /** + * Gets the observation count by group for given survey IDs + * + * @param {number[]} surveyIds - Array of survey IDs + * @param {string[]} groupByColumns - Columns to group by + * @param {string[]} groupByQuantitativeMeasurements - Quantitative measurements to group by + * @param {string[]} groupByQualitativeMeasurements - Qualitative measurements to group by + * @returns {Promise} - Observation count by group + * @memberof AnalyticsRepository + */ + async getObservationCountByGroup( + surveyIds: number[], + groupByColumns: string[], + groupByQuantitativeMeasurements: string[], + groupByQualitativeMeasurements: string[] + ): Promise { + const knex = getKnex(); + + const combinedColumns = [...groupByColumns, ...groupByQuantitativeMeasurements, ...groupByQualitativeMeasurements]; + + // Subquery to get the total count, used for calculating ratios + const totalCountSubquery = knex('observation_subcount as os') + .sum('os.subcount as total') + .leftJoin('survey_observation as so', 'so.survey_observation_id', 'os.survey_observation_id') + .whereIn('so.survey_id', surveyIds) + .first() + .toString(); + + // Create columns for quantitative measurements + const quantColumns = groupByQuantitativeMeasurements.map((id) => + knex.raw(`MAX(CASE WHEN quant.critterbase_taxon_measurement_id = ? THEN quant.value END) as ??`, [id, id]) + ); + + // Create columns for qualitative measurements + const qualColumns = groupByQualitativeMeasurements.map((id) => + knex.raw( + `STRING_AGG(DISTINCT CASE WHEN qual.critterbase_taxon_measurement_id = ? THEN qual.critterbase_measurement_qualitative_option_id::text END, ',') as ??`, + [id, id] + ) + ); + + const queryBuilder = knex + .with('temp_observations', (qb) => { + qb.select( + 'os.subcount', + 'os.observation_subcount_id', + 'so.survey_id', + ...groupByColumns.map((column) => knex.raw('??', [column])), + ...quantColumns, + ...qualColumns + ) + .from('observation_subcount as os') + .leftJoin('survey_observation as so', 'so.survey_observation_id', 'os.survey_observation_id') + .leftJoin( + 'observation_subcount_qualitative_measurement as qual', + 'qual.observation_subcount_id', + 'os.observation_subcount_id' + ) + .leftJoin( + 'observation_subcount_quantitative_measurement as quant', + 'quant.observation_subcount_id', + 'os.observation_subcount_id' + ) + .whereIn('so.survey_id', surveyIds) + .groupBy('os.subcount', 'os.observation_subcount_id', 'so.survey_id', ...groupByColumns); + }) + .select(knex.raw('public.gen_random_uuid() as id')) // Generate a unique ID for the row + .select(knex.raw('COUNT(subcount)::NUMERIC as row_count')) + .select(knex.raw('SUM(subcount)::NUMERIC as individual_count')) + .select(knex.raw(`ROUND(SUM(os.subcount)::NUMERIC / (${totalCountSubquery}) * 100, 2) as individual_percentage`)) + .select(groupByColumns.map((column) => knex.raw('??', [column]))) + // Measurement properties are objects of {'' : '', '' : ''} + .select( + knex.raw( + `jsonb_build_object(${groupByQuantitativeMeasurements + .map((column) => `'${column}', ??`) + .join(', ')}) as quant_measurements`, + groupByQuantitativeMeasurements + ) + ) + .select( + knex.raw( + `jsonb_build_object(${groupByQualitativeMeasurements + .map((column) => `'${column}', ??`) + .join(', ')}) as qual_measurements`, + groupByQualitativeMeasurements + ) + ) + .from('temp_observations as os') + .groupBy(combinedColumns) + .orderBy('individual_count', 'desc'); + + const response = await this.connection.knex(queryBuilder, ObservationCountByGroupSQLResponse); + + return response.rows; + } +} diff --git a/api/src/services/analytics-service.test.ts b/api/src/services/analytics-service.test.ts new file mode 100644 index 0000000000..ccd67465f6 --- /dev/null +++ b/api/src/services/analytics-service.test.ts @@ -0,0 +1,518 @@ +import chai, { expect } from 'chai'; +import { describe } from 'mocha'; +import sinon from 'sinon'; +import sinonChai from 'sinon-chai'; +import { + ObservationAnalyticsResponse, + ObservationCountByGroup, + ObservationCountByGroupSQLResponse, + ObservationCountByGroupWithMeasurements +} from '../models/observation-analytics'; +import { AnalyticsRepository } from '../repositories/analytics-repository'; +import { getMockDBConnection } from '../__mocks__/db'; +import { AnalyticsService } from './analytics-service'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition, + CritterbaseService +} from './critterbase-service'; + +chai.use(sinonChai); + +describe('AnalyticsService', () => { + afterEach(() => { + sinon.restore(); + }); + + describe('getObservationCountByGroup', () => { + it('returns an array of observation count analytics records', async () => { + const dbConnection = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnection); + + const mockGetObservationCountByGroupResponse: ObservationCountByGroupSQLResponse[] = [ + { + id: '123-456-789', + row_count: 3, + individual_count: 62, + individual_percentage: 57.41, + survey_sample_site_id: 4, + survey_sample_period_id: 4, + quant_measurements: {}, + qual_measurements: { '337f67fa-296d-43a9-88b2-ffdc77891aee': '61d1532d-b06e-4300-8da6-d195cc98f34e' } + }, + { + id: '987-654-321', + row_count: 2, + individual_count: 46, + individual_percentage: 42.59, + survey_sample_site_id: 4, + survey_sample_period_id: 4, + quant_measurements: {}, + qual_measurements: { '337f67fa-296d-43a9-88b2-ffdc77891aee': 'dd9a1672-ac93-4598-b166-caad463ed6f2' } + } + ]; + + sinon + .stub(AnalyticsRepository.prototype, 'getObservationCountByGroup') + .resolves(mockGetObservationCountByGroupResponse); + + sinon.stub(CritterbaseService.prototype, 'getQualitativeMeasurementTypeDefinition').resolves([ + { + taxon_measurement_id: '337f67fa-296d-43a9-88b2-ffdc77891aee', + itis_tsn: 180692, + measurement_name: 'antler configuration', + measurement_desc: null, + options: [ + { + qualitative_option_id: '61d1532d-b06e-4300-8da6-d195cc98f34e', + option_label: 'less than 3 points', + option_value: 0, + option_desc: null + }, + { + qualitative_option_id: 'dd9a1672-ac93-4598-b166-caad463ed6f2', + option_label: 'more than 3 points', + option_value: 1, + option_desc: null + } + ] + } + ]); + + sinon.stub(CritterbaseService.prototype, 'getQuantitativeMeasurementTypeDefinition').resolves([]); + + const surveyIds = [4]; + const groupByColumns = ['survey_sample_site_id', 'survey_sample_period_id']; + const groupByQuantitativeMeasurements: string[] = []; + const groupByQualitativeMeasurements = ['337f67fa-296d-43a9-88b2-ffdc77891aee']; + + const response = await analyticsService.getObservationCountByGroup( + surveyIds, + groupByColumns, + groupByQuantitativeMeasurements, + groupByQualitativeMeasurements + ); + + const expectedResponse: ObservationAnalyticsResponse[] = [ + { + id: '123-456-789', + row_count: 3, + individual_count: 62, + individual_percentage: 57.41, + qualitative_measurements: [ + { + option: { option_id: '61d1532d-b06e-4300-8da6-d195cc98f34e', option_label: 'less than 3 points' }, + taxon_measurement_id: '337f67fa-296d-43a9-88b2-ffdc77891aee', + measurement_name: 'antler configuration' + } + ], + quantitative_measurements: [], + survey_sample_period_id: 4, + survey_sample_site_id: 4 + }, + { + id: '987-654-321', + row_count: 2, + individual_count: 46, + individual_percentage: 42.59, + qualitative_measurements: [ + { + option: { option_id: 'dd9a1672-ac93-4598-b166-caad463ed6f2', option_label: 'more than 3 points' }, + taxon_measurement_id: '337f67fa-296d-43a9-88b2-ffdc77891aee', + measurement_name: 'antler configuration' + } + ], + quantitative_measurements: [], + survey_sample_period_id: 4, + survey_sample_site_id: 4 + } + ]; + + expect(response).to.eql(expectedResponse); + }); + }); + + describe('_filterNonEmptyColumns', () => { + it('returns an array of non-empty columns', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const columns = ['a', '', 'b', 'c', '']; + + const result = analyticsService._filterNonEmptyColumns(columns); + + expect(result).to.eql(['a', 'b', 'c']); + }); + }); + + describe('_fetchQualitativeDefinitions', () => { + it('returns an array of qualitative measurement type definitions', async () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const mockQualitativeMeasurementTypeDefinitions: CBQualitativeMeasurementTypeDefinition[] = [ + { + itis_tsn: 123456, + taxon_measurement_id: '1', + measurement_name: 'measurement_name', + measurement_desc: 'measurement_desc', + options: [ + { + qualitative_option_id: '3', + option_label: 'option_label', + option_value: 0, + option_desc: 'option_desc' + } + ] + } + ]; + + sinon + .stub(CritterbaseService.prototype, 'getQualitativeMeasurementTypeDefinition') + .resolves(mockQualitativeMeasurementTypeDefinitions); + + const counts: ObservationCountByGroupSQLResponse[] = [ + { + id: '123-456-789', + row_count: 1, + individual_count: 1, + individual_percentage: 1, + life_stage: 'adult', + antler_length: 20, + quant_measurements: { + '1': 1 + }, + qual_measurements: { + '2': '2' + } + } + ]; + + const result = await analyticsService._fetchQualitativeDefinitions(counts); + + expect(result).to.eql(mockQualitativeMeasurementTypeDefinitions); + }); + }); + + describe('_fetchQuantitativeDefinitions', () => { + it('returns an array of quantitative measurement type definitions', async () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const mockQuantitativeMeasurementTypeDefinitions: CBQuantitativeMeasurementTypeDefinition[] = [ + { + itis_tsn: 123456, + taxon_measurement_id: '1', + measurement_name: 'measurement_name', + measurement_desc: 'measurement_desc', + min_value: 1, + max_value: 2, + unit: 'millimeter' + } + ]; + + sinon + .stub(CritterbaseService.prototype, 'getQuantitativeMeasurementTypeDefinition') + .resolves(mockQuantitativeMeasurementTypeDefinitions); + + const counts: ObservationCountByGroupSQLResponse[] = [ + { + id: '123-456-789', + row_count: 5, + individual_count: 10, + individual_percentage: 46, + life_stage: 'adult', + antler_length: 20, + quant_measurements: { + '1': 1 + }, + qual_measurements: { + '2': '2' + } + } + ]; + + const result = await analyticsService._fetchQuantitativeDefinitions(counts); + + expect(result).to.eql(mockQuantitativeMeasurementTypeDefinitions); + }); + }); + + describe('_getQualitativeMeasurementIds', () => { + it('returns an array of measurement IDs', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const counts: ObservationCountByGroupSQLResponse[] = [ + { + id: '123-456-789', + row_count: 1, + individual_count: 1, + individual_percentage: 1, + life_stage: 'adult', + antler_length: 20, + quant_measurements: { + '1': 1, + '2': 2 + }, + qual_measurements: { + '3': '3', + '4': '4' + } + } + ]; + + const result = analyticsService._getQualitativeMeasurementIds(counts); + + expect(result).to.eql(['3', '4']); + }); + }); + + describe('_getQuantitativeMeasurementIds', () => { + it('returns an array of measurement IDs', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const counts: ObservationCountByGroupSQLResponse[] = [ + { + id: '123-456-789', + row_count: 1, + individual_count: 1, + individual_percentage: 1, + life_stage: 'adult', + antler_length: 20, + quant_measurements: { + '1': 1, + '2': 2 + }, + qual_measurements: { + '3': '3', + '4': '4' + } + } + ]; + + const result = analyticsService._getQuantitativeMeasurementIds(counts); + + expect(result).to.eql(['1', '2']); + }); + }); + + describe('_processCounts', () => { + it('returns an array of observation analytics responses', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const counts: (ObservationCountByGroupWithMeasurements & ObservationCountByGroup)[] = [ + { + row_count: 1, + individual_count: 1, + individual_percentage: 1, + quant_measurements: [ + { + critterbase_taxon_measurement_id: '1', + value: 1 + } + ], + qual_measurements: [ + { + critterbase_taxon_measurement_id: '2', + option_id: '3' + } + ] + } + ]; + + const qualitativeDefinitions: CBQualitativeMeasurementTypeDefinition[] = []; + const quantitativeDefinitions: CBQuantitativeMeasurementTypeDefinition[] = []; + + const result = analyticsService._processCounts(counts, qualitativeDefinitions, quantitativeDefinitions); + + expect(result).to.be.an('array'); + expect(result.length).to.equal(1); + }); + }); + + describe('_mapQualitativeMeasurements', () => { + it('returns an array of qualitative measurement analytics', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const qualMeasurements = [ + { + critterbase_taxon_measurement_id: '11', + option_id: '1' + }, + { + critterbase_taxon_measurement_id: '22', + option_id: null + } + ]; + + const definitions: CBQualitativeMeasurementTypeDefinition[] = [ + { + itis_tsn: 123456, + taxon_measurement_id: '11', + measurement_name: 'measurement_name', + measurement_desc: 'measurement_desc', + options: [ + { + qualitative_option_id: '1', + option_label: 'option_label', + option_value: 1, + option_desc: 'option_desc' + } + ] + }, + { + itis_tsn: 123456, + taxon_measurement_id: '22', + measurement_name: 'measurement_name', + measurement_desc: 'measurement_desc', + options: [ + { + qualitative_option_id: '2', + option_label: 'option_label', + option_value: 2, + option_desc: 'option_desc' + } + ] + } + ]; + + const result = analyticsService._mapQualitativeMeasurements(qualMeasurements, definitions); + + expect(result).to.eql([ + { + taxon_measurement_id: '11', + measurement_name: 'measurement_name', + option: { + option_id: '1', + option_label: 'option_label' + } + }, + { + taxon_measurement_id: '22', + measurement_name: 'measurement_name', + option: { + option_id: null, + option_label: '' + } + } + ]); + }); + }); + + describe('_mapQuantitativeMeasurements', () => { + it('returns an array of quantitative measurement analytics', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const quantMeasurements = [ + { + critterbase_taxon_measurement_id: '11', + value: 1 + }, + { + critterbase_taxon_measurement_id: '22', + value: null + } + ]; + + const definitions: CBQuantitativeMeasurementTypeDefinition[] = [ + { + itis_tsn: 123456, + taxon_measurement_id: '11', + measurement_name: 'measurement_name', + measurement_desc: 'measurement_desc', + min_value: 1, + max_value: 2, + unit: 'millimeter' + }, + { + itis_tsn: 123456, + taxon_measurement_id: '22', + measurement_name: 'measurement_name', + measurement_desc: 'measurement_desc', + min_value: 1, + max_value: 2, + unit: 'centimeter' + } + ]; + + const result = analyticsService._mapQuantitativeMeasurements(quantMeasurements, definitions); + + expect(result).to.eql([ + { + taxon_measurement_id: '11', + measurement_name: 'measurement_name', + value: 1 + }, + { + measurement_name: 'measurement_name', + taxon_measurement_id: '22', + value: null + } + ]); + }); + }); + + describe('_transformMeasurementObjectKeysToArrays', () => { + it('returns an array of transformed observation counts', () => { + const dbConnectionObj = getMockDBConnection(); + + const analyticsService = new AnalyticsService(dbConnectionObj); + + const counts: ObservationCountByGroupSQLResponse[] = [ + { + id: '123-456-789', + row_count: 1, + individual_count: 1, + individual_percentage: 1, + life_stage: 'adult', + antler_length: 20, + quant_measurements: { + '1': 1 + }, + qual_measurements: { + '2': '2' + } + } + ]; + + const result = analyticsService._transformMeasurementObjectKeysToArrays(counts); + + expect(result).to.eql([ + { + id: '123-456-789', + row_count: 1, + individual_count: 1, + individual_percentage: 1, + qual_measurements: [ + { + critterbase_taxon_measurement_id: '2', + option_id: '2' + } + ], + quant_measurements: [ + { + critterbase_taxon_measurement_id: '1', + value: 1 + } + ], + antler_length: 20, + life_stage: 'adult' + } + ]); + }); + }); +}); diff --git a/api/src/services/analytics-service.ts b/api/src/services/analytics-service.ts new file mode 100644 index 0000000000..06cf779344 --- /dev/null +++ b/api/src/services/analytics-service.ts @@ -0,0 +1,282 @@ +import { IDBConnection } from '../database/db'; +import { + ObservationAnalyticsResponse, + ObservationCountByGroup, + ObservationCountByGroupSQLResponse, + ObservationCountByGroupWithMeasurements, + QualitativeMeasurementAnalytics, + QuantitativeMeasurementAnalytics +} from '../models/observation-analytics'; +import { AnalyticsRepository } from '../repositories/analytics-repository'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition, + CritterbaseService +} from './critterbase-service'; +import { DBService } from './db-service'; + +/** + * Handles all business logic related to data analytics. + * + * @export + * @class AnalyticsService + * @extends {DBService} + */ +export class AnalyticsService extends DBService { + analyticsRepository: AnalyticsRepository; + + critterbaseService: CritterbaseService; + + constructor(connection: IDBConnection) { + super(connection); + this.analyticsRepository = new AnalyticsRepository(connection); + + this.critterbaseService = new CritterbaseService({ + keycloak_guid: this.connection.systemUserGUID(), + username: this.connection.systemUserIdentifier() + }); + } + + /** + * Gets observation counts by group for given survey IDs and groupings. + * + * @param {number[]} surveyIds Array of survey IDs + * @param {string[]} groupByColumns Columns to group by + * @param {string[]} groupByQuantitativeMeasurements Quantitative measurements to group by + * @param {string[]} groupByQualitativeMeasurements Qualitative measurements to group by + * @return {Promise} Array of ObservationCountByGroupWithNamedMeasurements + * @memberof AnalyticsService + */ + async getObservationCountByGroup( + surveyIds: number[], + groupByColumns: string[], + groupByQuantitativeMeasurements: string[], + groupByQualitativeMeasurements: string[] + ): Promise { + // Fetch observation counts from repository + const counts = await this.analyticsRepository.getObservationCountByGroup( + surveyIds, + this._filterNonEmptyColumns(groupByColumns), + this._filterNonEmptyColumns(groupByQuantitativeMeasurements), + this._filterNonEmptyColumns(groupByQualitativeMeasurements) + ); + + // Fetch measurement definitions in parallel + const [qualitativeDefinitions, quantitativeDefinitions] = await Promise.all([ + this._fetchQualitativeDefinitions(counts), + this._fetchQuantitativeDefinitions(counts) + ]); + + const transformedCounts = this._transformMeasurementObjectKeysToArrays(counts); + + return this._processCounts(transformedCounts, qualitativeDefinitions, quantitativeDefinitions); + } + + /** + * Filters out empty columns. + * + * @param {string[]} columns + * @return {*} {string[]} + * @memberof AnalyticsService + */ + _filterNonEmptyColumns(columns: string[]): string[] { + return columns.filter((column) => column.trim() !== ''); + } + + /** + * Fetches qualitative measurement definitions for given counts. + * + * @param {ObservationCountByGroupSQLResponse[]} counts + * @return {*} {Promise} + * @memberof AnalyticsService + */ + async _fetchQualitativeDefinitions( + counts: ObservationCountByGroupSQLResponse[] + ): Promise { + const qualTaxonMeasurementIds = this._getQualitativeMeasurementIds(counts); + + if (qualTaxonMeasurementIds.length === 0) { + return []; + } + + return this.critterbaseService.getQualitativeMeasurementTypeDefinition(qualTaxonMeasurementIds); + } + + /** + * Fetches quantitative measurement definitions for given counts. + * + * @param {ObservationCountByGroupSQLResponse[]} counts + * @return {*} {Promise} + * @memberof AnalyticsService + */ + async _fetchQuantitativeDefinitions( + counts: ObservationCountByGroupSQLResponse[] + ): Promise { + const quantTaxonMeasurementIds = this._getQuantitativeMeasurementIds(counts); + + if (quantTaxonMeasurementIds.length === 0) { + return []; + } + + return this.critterbaseService.getQuantitativeMeasurementTypeDefinition(quantTaxonMeasurementIds); + } + + /** + * Returns array of unique qualitative measurement IDs from given counts. + * + * @param {ObservationCountByGroupSQLResponse[]} counts + * @return {*} {string[]} + * @memberof AnalyticsService + */ + _getQualitativeMeasurementIds(counts: ObservationCountByGroupSQLResponse[]): string[] { + return Array.from(new Set(counts.flatMap((count) => Object.keys(count.qual_measurements)))); + } + + /** + * Returns array of unique quantitative measurement IDs from given counts. + * + * @param {ObservationCountByGroupSQLResponse[]} counts + * @return {*} {string[]} + * @memberof AnalyticsService + */ + _getQuantitativeMeasurementIds(counts: ObservationCountByGroupSQLResponse[]): string[] { + return Array.from(new Set(counts.flatMap((count) => Object.keys(count.quant_measurements)))); + } + + /** + * Parses the raw counts object, stripping out extra fields, and maps measurements to their definitions. + * + * @param {((ObservationCountByGroupWithMeasurements & ObservationCountByGroup)[])} counts + * @param {CBQualitativeMeasurementTypeDefinition[]} qualitativeDefinitions + * @param {CBQuantitativeMeasurementTypeDefinition[]} quantitativeDefinitions + * @return {*} {ObservationAnalyticsResponse[]} + * @memberof AnalyticsService + */ + _processCounts( + counts: (ObservationCountByGroupWithMeasurements & ObservationCountByGroup)[], + qualitativeDefinitions: CBQualitativeMeasurementTypeDefinition[], + quantitativeDefinitions: CBQuantitativeMeasurementTypeDefinition[] + ): ObservationAnalyticsResponse[] { + const newCounts: ObservationAnalyticsResponse[] = []; + + for (const count of counts) { + const { row_count, individual_count, individual_percentage, qual_measurements, quant_measurements, ...rest } = + count; + + newCounts.push({ + row_count, + individual_count, + individual_percentage, + ...rest, + qualitative_measurements: this._mapQualitativeMeasurements(qual_measurements, qualitativeDefinitions), + quantitative_measurements: this._mapQuantitativeMeasurements(quant_measurements, quantitativeDefinitions) + }); + } + + return newCounts; + } + + /** + * Maps qualitative measurements to their definitions. + * + * @param {({ option_id: string | null; critterbase_taxon_measurement_id: string }[])} qualMeasurements + * @param {CBQualitativeMeasurementTypeDefinition[]} definitions + * @return {*} {QualitativeMeasurementAnalytics[]} + * @memberof AnalyticsService + */ + _mapQualitativeMeasurements( + qualMeasurements: { option_id: string | null; critterbase_taxon_measurement_id: string }[], + definitions: CBQualitativeMeasurementTypeDefinition[] + ): QualitativeMeasurementAnalytics[] { + return qualMeasurements + .map((measurement) => { + const definition = definitions.find( + (def) => def.taxon_measurement_id === measurement.critterbase_taxon_measurement_id + ); + + if (!definition) { + return null; + } + + return { + taxon_measurement_id: measurement.critterbase_taxon_measurement_id, + measurement_name: definition.measurement_name ?? '', + option: { + option_id: measurement.option_id, + option_label: + definition.options.find((option) => option.qualitative_option_id === measurement.option_id) + ?.option_label ?? '' + } + }; + }) + .filter((item): item is QualitativeMeasurementAnalytics => item !== null); + } + + /** + * Maps quantitative measurements to their definitions. + * + * @param {({ value: number | null; critterbase_taxon_measurement_id: string }[])} quantMeasurements + * @param {CBQuantitativeMeasurementTypeDefinition[]} definitions + * @return {*} {QuantitativeMeasurementAnalytics[]} + * @memberof AnalyticsService + */ + _mapQuantitativeMeasurements( + quantMeasurements: { value: number | null; critterbase_taxon_measurement_id: string }[], + definitions: CBQuantitativeMeasurementTypeDefinition[] + ): QuantitativeMeasurementAnalytics[] { + return quantMeasurements + .map((measurement) => { + const definition = definitions.find( + (def) => def.taxon_measurement_id === measurement.critterbase_taxon_measurement_id + ); + + if (!definition) { + return null; + } + + return { + taxon_measurement_id: measurement.critterbase_taxon_measurement_id, + measurement_name: definition.measurement_name ?? '', + value: measurement.value + }; + }) + .filter((item): item is QuantitativeMeasurementAnalytics => item !== null); + } + + /** + * Transforms the keys of the measurement objects to arrays. + * + * @param {ObservationCountByGroupSQLResponse[]} counts + * @return {*} {((ObservationCountByGroupWithMeasurements & ObservationCountByGroup)[])} + * @memberof AnalyticsService + */ + _transformMeasurementObjectKeysToArrays( + counts: ObservationCountByGroupSQLResponse[] + ): (ObservationCountByGroupWithMeasurements & ObservationCountByGroup)[] { + return counts.map((count) => { + const { row_count, individual_count, individual_percentage, quant_measurements, qual_measurements, ...rest } = + count; + + // Transform quantitative measurements + const quantitative = Object.entries(quant_measurements).map(([measurementId, value]) => ({ + critterbase_taxon_measurement_id: measurementId, + value: value + })); + + // Transform qualitative measurements + const qualitative = Object.entries(qual_measurements).map(([measurementId, optionId]) => ({ + critterbase_taxon_measurement_id: measurementId, + option_id: optionId + })); + + return { + row_count, + individual_count, + individual_percentage, + ...rest, + qual_measurements: qualitative, + quant_measurements: quantitative + }; + }); + } +} diff --git a/app/src/features/surveys/view/SurveyPage.tsx b/app/src/features/surveys/view/SurveyPage.tsx index 472f17a0cb..a1e7c05aac 100644 --- a/app/src/features/surveys/view/SurveyPage.tsx +++ b/app/src/features/surveys/view/SurveyPage.tsx @@ -39,9 +39,11 @@ const SurveyPage: React.FC = () => { - - - + + + + + diff --git a/app/src/features/surveys/view/components/analytics/SurveyObservationAnalytics.tsx b/app/src/features/surveys/view/components/analytics/SurveyObservationAnalytics.tsx new file mode 100644 index 0000000000..a12191b6b1 --- /dev/null +++ b/app/src/features/surveys/view/components/analytics/SurveyObservationAnalytics.tsx @@ -0,0 +1,194 @@ +import Box from '@mui/material/Box'; +import Button from '@mui/material/Button'; +import Checkbox from '@mui/material/Checkbox'; +import grey from '@mui/material/colors/grey'; +import Divider from '@mui/material/Divider'; +import Stack from '@mui/material/Stack'; +import ToggleButton from '@mui/material/ToggleButton'; +import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; +import Typography from '@mui/material/Typography'; +import { LoadingGuard } from 'components/loading/LoadingGuard'; +import { SkeletonTable } from 'components/loading/SkeletonLoaders'; +import { ObservationAnalyticsDataTableContainer } from 'features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer'; +import { useBiohubApi } from 'hooks/useBioHubApi'; +import { useSurveyContext } from 'hooks/useContext'; +import useDataLoader from 'hooks/useDataLoader'; +import startCase from 'lodash-es/startCase'; +import { useEffect, useState } from 'react'; +import { ObservationAnalyticsNoDataOverlay } from './components/ObservationAnalyticsNoDataOverlay'; +type GroupByColumnType = 'column' | 'quantitative_measurement' | 'qualitative_measurement'; + +export type IGroupByOption = { + label: string; + field: string; + type: GroupByColumnType; +}; + +const initialGroupByColumnOptions: IGroupByOption[] = [ + { label: 'Sampling Site', field: 'survey_sample_site_id', type: 'column' } +]; + +const allGroupByColumnOptions: IGroupByOption[] = [ + ...initialGroupByColumnOptions, + { label: 'Sampling Method', field: 'survey_sample_method_id', type: 'column' }, + { label: 'Sampling Period', field: 'survey_sample_period_id', type: 'column' }, + { label: 'Species', field: 'itis_tsn', type: 'column' }, + { label: 'Date', field: 'observation_date', type: 'column' } +]; + +export const SurveyObservationAnalytics = () => { + const biohubApi = useBiohubApi(); + + const { surveyId, projectId } = useSurveyContext(); + + const [groupByColumns, setGroupByColumns] = useState(initialGroupByColumnOptions); + const [groupByQualitativeMeasurements, setGroupByQualitativeMeasurements] = useState([]); + const [groupByQuantitativeMeasurements, setGroupByQuantitativeMeasurements] = useState([]); + + const measurementDefinitionsDataLoader = useDataLoader(() => + biohubApi.observation.getObservationMeasurementDefinitions(projectId, surveyId) + ); + + useEffect(() => { + measurementDefinitionsDataLoader.load(); + }, [measurementDefinitionsDataLoader]); + + const groupByOptions: IGroupByOption[] = [ + ...allGroupByColumnOptions, + ...(measurementDefinitionsDataLoader.data?.qualitative_measurements.map((measurement) => ({ + label: startCase(measurement.measurement_name), + field: measurement.taxon_measurement_id, + type: 'qualitative_measurement' as GroupByColumnType + })) ?? []), + ...(measurementDefinitionsDataLoader.data?.quantitative_measurements.map((measurement) => ({ + label: startCase(measurement.measurement_name), + field: measurement.taxon_measurement_id, + type: 'quantitative_measurement' as GroupByColumnType + })) ?? []) + ]; + + const handleToggleChange = (_: React.MouseEvent, value: IGroupByOption[]) => { + if (!value[0]?.type) return; + + // Update group by arrays + if (value[0].type === 'column') { + updateGroupBy(value[0], setGroupByColumns); + } + if (value[0].type === 'qualitative_measurement') { + updateGroupBy(value[0], setGroupByQualitativeMeasurements); + } + if (value[0].type === 'quantitative_measurement') { + updateGroupBy(value[0], setGroupByQuantitativeMeasurements); + } + }; + + const updateGroupBy = (value: IGroupByOption, setGroupBy: React.Dispatch>) => + setGroupBy((groupBy) => + groupBy.some((item) => item.field === value.field) + ? groupBy.filter((item) => item.field !== value.field) + : [...groupBy, value] + ); + + const allGroupByColumns = [...groupByColumns, ...groupByQualitativeMeasurements, ...groupByQuantitativeMeasurements]; + + return ( + + + + + } + isLoadingFallbackDelay={100}> + + {/* Group by header */} + + + GROUP BY + + + + + {/* Render toggle buttons for each group by option */} + {groupByOptions.map((option) => ( + item.field === option.field) || + groupByQualitativeMeasurements.some((item) => item.field === option.field) || + groupByQuantitativeMeasurements.some((item) => item.field === option.field) + }> + + item.field === option.field) || + groupByQualitativeMeasurements.some((item) => item.field === option.field) || + groupByQuantitativeMeasurements.some((item) => item.field === option.field) + } + /> + {option.label} + + + ))} + + + + + + + {/* Overlay for when no group by columns are selected */} + {allGroupByColumns.length === 0 && !measurementDefinitionsDataLoader.isLoading && ( + + )} + + {/* Data grid displaying fetched data */} + {measurementDefinitionsDataLoader.data && allGroupByColumns.length > 0 && ( + + )} + + + ); +}; diff --git a/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTable.tsx b/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTable.tsx new file mode 100644 index 0000000000..2fed674685 --- /dev/null +++ b/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTable.tsx @@ -0,0 +1,68 @@ +import { GridColDef, GridColumnVisibilityModel } from '@mui/x-data-grid'; +import { StyledDataGrid } from 'components/data-grid/StyledDataGrid'; +import { IObservationAnalyticsRow } from 'features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer'; + +const rowHeight = 50; + +interface IObservationAnalyticsDataTableProps { + isLoading: boolean; + columns: GridColDef[]; + rows: IObservationAnalyticsRow[]; + columnVisibilityModel: GridColumnVisibilityModel; +} + +/** + * Observation Analytics Data Table. + * + * @param {IObservationAnalyticsDataTableProps} props + * @return {*} + */ +export const ObservationAnalyticsDataTable = (props: IObservationAnalyticsDataTableProps) => { + const { isLoading, columns, rows, columnVisibilityModel } = props; + + return ( + + ); +}; diff --git a/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer.tsx b/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer.tsx new file mode 100644 index 0000000000..b5e09248f1 --- /dev/null +++ b/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer.tsx @@ -0,0 +1,154 @@ +import { GridColumnVisibilityModel } from '@mui/x-data-grid'; +import { ObservationAnalyticsDataTable } from 'features/surveys/view/components/analytics/components/ObservationAnalyticsDataTable'; +import { IGroupByOption } from 'features/surveys/view/components/analytics/SurveyObservationAnalytics'; +import { useBiohubApi } from 'hooks/useBioHubApi'; +import { useSurveyContext, useTaxonomyContext } from 'hooks/useContext'; +import useDataLoader from 'hooks/useDataLoader'; +import { IObservationCountByGroup } from 'interfaces/useAnalyticsApi.interface'; +import { useEffect, useMemo } from 'react'; +import { + getBasicGroupByColDefs, + getDateColDef, + getIndividualCountColDef, + getIndividualPercentageColDef, + getRowCountColDef, + getSamplingMethodColDef, + getSamplingPeriodColDef, + getSamplingSiteColDef, + getSpeciesColDef +} from './ObservationsAnalyticsGridColumnDefinitions'; + +export type IObservationAnalyticsRow = Omit< + IObservationCountByGroup, + 'quantitative_measurements' | 'qualitative_measurements' +> & { + [key: string]: string | number | null; +}; + +// Base columns that are always displayed, and not part of the group by columns +const BaseColumns = ['row_count', 'individual_count', 'individual_percentage']; + +interface IObservationAnalyticsDataTableContainerProps { + groupByColumns: IGroupByOption[]; + groupByQuantitativeMeasurements: IGroupByOption[]; + groupByQualitativeMeasurements: IGroupByOption[]; +} + +/** + * Observation Analytics Data Table Container. + * Fetches and parses the observation analytics data and passes it to the ObservationAnalyticsDataTable component. + * + * @param {IObservationAnalyticsDataTableContainerProps} props + * @return {*} + */ +export const ObservationAnalyticsDataTableContainer = (props: IObservationAnalyticsDataTableContainerProps) => { + const { groupByColumns, groupByQuantitativeMeasurements, groupByQualitativeMeasurements } = props; + + const biohubApi = useBiohubApi(); + const surveyContext = useSurveyContext(); + const taxonomyContext = useTaxonomyContext(); + + const analyticsDataLoader = useDataLoader( + ( + surveyId: number, + groupByColumns: IGroupByOption[], + groupByQuantitativeMeasurements: IGroupByOption[], + groupByQualitativeMeasurements: IGroupByOption[] + ) => + biohubApi.analytics.getObservationCountByGroup( + [surveyId], + groupByColumns.map((item) => item.field), + groupByQuantitativeMeasurements.map((item) => item.field), + groupByQualitativeMeasurements.map((item) => item.field) + ) + ); + + useEffect( + () => { + analyticsDataLoader.refresh( + surveyContext.surveyId, + groupByColumns, + groupByQuantitativeMeasurements, + groupByQualitativeMeasurements + ); + }, + // eslint-disable-next-line + [groupByColumns, groupByQualitativeMeasurements, groupByQuantitativeMeasurements, surveyContext.surveyId] + ); + + const rows = useMemo( + () => + analyticsDataLoader?.data?.map((row) => { + const { quantitative_measurements, qualitative_measurements, ...nonMeasurementRows } = row; + + const newRow: IObservationAnalyticsRow = nonMeasurementRows; + + qualitative_measurements.forEach((measurement) => { + newRow[measurement.taxon_measurement_id] = measurement.option.option_label; + }); + + quantitative_measurements.forEach((measurement) => { + newRow[measurement.taxon_measurement_id] = measurement.value; + }); + + return newRow; + }) ?? [], + [analyticsDataLoader?.data] + ); + + const sampleSites = useMemo( + () => surveyContext.sampleSiteDataLoader.data?.sampleSites ?? [], + [surveyContext.sampleSiteDataLoader.data?.sampleSites] + ); + + const allGroupByColumns = useMemo( + () => [...groupByColumns, ...groupByQualitativeMeasurements, ...groupByQuantitativeMeasurements], + [groupByColumns, groupByQualitativeMeasurements, groupByQuantitativeMeasurements] + ); + + const columns = useMemo( + () => [ + getRowCountColDef(), + getIndividualCountColDef(), + getIndividualPercentageColDef(), + getSamplingSiteColDef(sampleSites), + getSamplingMethodColDef(sampleSites), + getSamplingPeriodColDef(sampleSites), + getSpeciesColDef(taxonomyContext.getCachedSpeciesTaxonomyById), + getDateColDef(), + ...getBasicGroupByColDefs([...groupByQualitativeMeasurements, ...groupByQuantitativeMeasurements]) + ], + // eslint-disable-next-line + [rows, allGroupByColumns] + ); + + const columnVisibilityModel = useMemo(() => { + const _columnVisibilityModel: GridColumnVisibilityModel = {}; + + for (const column of columns) { + // Set all columns to visible by default + _columnVisibilityModel[column.field] = true; + + if (BaseColumns.includes(column.field)) { + // Don't hide base columns + continue; + } + + if (!allGroupByColumns.some((item) => item.field === column.field)) { + // Set columns that are not part of the group by columns (not selected in the UI) to hidden + _columnVisibilityModel[column.field] = false; + } + } + + return _columnVisibilityModel; + }, [allGroupByColumns, columns]); + + return ( + + ); +}; diff --git a/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsNoDataOverlay.tsx b/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsNoDataOverlay.tsx new file mode 100644 index 0000000000..f56d03e844 --- /dev/null +++ b/app/src/features/surveys/view/components/analytics/components/ObservationAnalyticsNoDataOverlay.tsx @@ -0,0 +1,49 @@ +import Box from '@mui/material/Box'; +import grey from '@mui/material/colors/grey'; +import Stack from '@mui/material/Stack'; +import Typography from '@mui/material/Typography'; + +/** + * Returns an overlay with instructions on how to use the analytics feature. + * + * @return {*} + */ +export const ObservationAnalyticsNoDataOverlay = () => { + return ( + + + Calculate sex ratios, demographics, and more + + + Choose fields to analyze + + + The group by options depend on which fields apply to your observations. To add options, such as life stage + and sex, add fields to your observations by configuring your observations table. + + + + + How the calculations work  + + + The number observations and individuals will be calculated for each group. For example, if you group by life + stage, the number of individuals belonging to each life stage category will be calculated. If you group by + multiple fields, such as life stage and sex, the number of individuals belonging to each life stage and sex + combination will be calculated. + + + + + ); +}; diff --git a/app/src/features/surveys/view/components/analytics/components/ObservationsAnalyticsGridColumnDefinitions.tsx b/app/src/features/surveys/view/components/analytics/components/ObservationsAnalyticsGridColumnDefinitions.tsx new file mode 100644 index 0000000000..b24ef04e98 --- /dev/null +++ b/app/src/features/surveys/view/components/analytics/components/ObservationsAnalyticsGridColumnDefinitions.tsx @@ -0,0 +1,221 @@ +import grey from '@mui/material/colors/grey'; +import Typography from '@mui/material/Typography'; +import { GridColDef } from '@mui/x-data-grid'; +import { DATE_FORMAT } from 'constants/dateTimeFormats'; +import dayjs from 'dayjs'; +import { ScientificNameTypography } from 'features/surveys/animals/components/ScientificNameTypography'; +import { IObservationAnalyticsRow } from 'features/surveys/view/components/analytics/components/ObservationAnalyticsDataTableContainer'; +import { IGroupByOption } from 'features/surveys/view/components/analytics/SurveyObservationAnalytics'; +import { IGetSampleLocationDetails } from 'interfaces/useSamplingSiteApi.interface'; +import { IPartialTaxonomy } from 'interfaces/useTaxonomyApi.interface'; +import isEqual from 'lodash-es/isEqual'; + +/** + * Get the column definition for the row count. + * + * @return {*} {GridColDef} + */ +export const getRowCountColDef = (): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'row_count', + headerName: 'Number of observations', + type: 'number', + flex: 1, + minWidth: 180 +}); + +/** + * Get the column definition for the individual count. + * + * @return {*} {GridColDef} + */ +export const getIndividualCountColDef = (): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'individual_count', + headerName: 'Number of individuals', + type: 'number', + flex: 1, + minWidth: 180 +}); + +/** + * Get the column definition for the individual percentage. + * + * @return {*} {GridColDef} + */ +export const getIndividualPercentageColDef = (): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'individual_percentage', + headerName: 'Percentage of individuals', + type: 'number', + flex: 1, + minWidth: 180, + renderCell: (params) => ( + + {params.row.individual_percentage}  + + % + + + ) +}); + +/** + * Get the column definition for the species. + * + * @param {((id: number) => IPartialTaxonomy | null)} getFunction + * @return {*} {GridColDef} + */ +export const getSpeciesColDef = ( + getFunction: (id: number) => IPartialTaxonomy | null +): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'itis_tsn', + headerName: 'Species', + minWidth: 180, + renderCell: (params) => { + if (!params.row.itis_tsn) { + return null; + } + + const species = getFunction(params.row.itis_tsn); + + return ; + } +}); + +/** + * Get the column definition for the sampling site. + * + * @param {IGetSampleLocationDetails[]} sampleSites + * @return {*} {GridColDef} + */ +export const getSamplingSiteColDef = ( + sampleSites: IGetSampleLocationDetails[] +): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'survey_sample_site_id', + headerName: 'Site', + minWidth: 180, + renderCell: (params) => { + if (!params.row.survey_sample_site_id) { + return null; + } + + const site = sampleSites.find((site) => isEqual(params.row.survey_sample_site_id, site.survey_sample_site_id)); + + if (!site) { + return null; + } + + return {site.name}; + } +}); + +/** + * Get the column definition for the sampling method. + * + * @param {IGetSampleLocationDetails[]} sampleSites + * @return {*} {GridColDef} + */ +export const getSamplingMethodColDef = ( + sampleSites: IGetSampleLocationDetails[] +): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'survey_sample_method_id', + headerName: 'Method', + minWidth: 180, + renderCell: (params) => { + if (!params.row.survey_sample_method_id) { + return null; + } + + const method = sampleSites + .flatMap((site) => site.sample_methods) + .find((method) => isEqual(params.row.survey_sample_method_id, method.survey_sample_method_id)); + + if (!method) { + return null; + } + + return {method.technique.name}; + } +}); + +/** + * Get the column definition for the sampling period. + * + * @param {IGetSampleLocationDetails[]} sampleSites + * @return {*} {GridColDef} + */ +export const getSamplingPeriodColDef = ( + sampleSites: IGetSampleLocationDetails[] +): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'survey_sample_period_id', + headerName: 'Period', + minWidth: 220, + renderCell: (params) => { + if (!params.row.survey_sample_period_id) { + return null; + } + + const period = sampleSites + .flatMap((site) => site.sample_methods) + .flatMap((method) => method.sample_periods) + .find((period) => isEqual(params.row.survey_sample_period_id, period.survey_sample_period_id)); + + if (!period) { + return null; + } + + return ( + + {dayjs(period.start_date).format(DATE_FORMAT.ShortMediumDateFormat)}– + {dayjs(period.end_date).format(DATE_FORMAT.ShortMediumDateFormat)} + + ); + } +}); + +/** + * Get the column definition for the date. + * + * @return {*} {GridColDef} + */ +export const getDateColDef = (): GridColDef => ({ + headerAlign: 'left', + align: 'left', + field: 'observation_date', + headerName: 'Date', + minWidth: 180, + renderCell: (params) => + params.row.observation_date ? ( + {dayjs(params.row.observation_date).format(DATE_FORMAT.MediumDateFormat)} + ) : null +}); + +/** + * Get basic group by column definitions for the provided group by options. + * + * @param {IGroupByOption[]} groupByOptions + * @return {*} {GridColDef[]} + */ +export const getBasicGroupByColDefs = (groupByOptions: IGroupByOption[]): GridColDef[] => { + if (!groupByOptions.length) { + return []; + } + + return groupByOptions.map((item) => ({ + field: item.field, + headerName: item.label, + minWidth: 180 + })); +}; diff --git a/app/src/features/surveys/view/components/data-container/SurveyObservationTabularDataContainer.tsx b/app/src/features/surveys/view/components/data-container/SurveyObservationTabularDataContainer.tsx new file mode 100644 index 0000000000..3fc75572f8 --- /dev/null +++ b/app/src/features/surveys/view/components/data-container/SurveyObservationTabularDataContainer.tsx @@ -0,0 +1,81 @@ +import { mdiChartBar, mdiTallyMark5 } from '@mdi/js'; +import Icon from '@mdi/react'; +import Box from '@mui/material/Box'; +import Button from '@mui/material/Button'; +import ToggleButton from '@mui/material/ToggleButton'; +import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; +import { useState } from 'react'; +import { SurveySpatialObservationTable } from '../../survey-spatial/components/observation/SurveySpatialObservationTable'; +import { SurveyObservationAnalytics } from '../analytics/SurveyObservationAnalytics'; + +export enum SurveyObservationTabularDataContainerViewEnum { + COUNTS = 'COUNTS', + ANALYTICS = 'ANALYTICS' +} + +interface ISurveyObservationTabularDataContainerProps { + isLoading: boolean; +} + +const SurveyObservationTabularDataContainer = (props: ISurveyObservationTabularDataContainerProps) => { + const { isLoading } = props; + + const [activeDataView, setActiveDataView] = useState( + SurveyObservationTabularDataContainerViewEnum.COUNTS + ); + + const views = [ + { label: 'Counts', value: SurveyObservationTabularDataContainerViewEnum.COUNTS, icon: mdiTallyMark5 }, + { label: 'Analytics', value: SurveyObservationTabularDataContainerViewEnum.ANALYTICS, icon: mdiChartBar } + ]; + + return ( + <> + + { + if (!view) { + // An active view must be selected at all times + return; + } + + setActiveDataView(view); + }} + exclusive + sx={{ + display: 'flex', + gap: 1, + '& Button': { + py: 0.25, + px: 1.5, + border: 'none', + borderRadius: '4px !important', + fontSize: '0.875rem', + fontWeight: 700, + letterSpacing: '0.02rem' + } + }}> + {views.map((view) => ( + } + value={view.value}> + {view.label} + + ))} + + + + {activeDataView === SurveyObservationTabularDataContainerViewEnum.COUNTS && ( + + )} + {activeDataView === SurveyObservationTabularDataContainerViewEnum.ANALYTICS && } + + + ); +}; + +export default SurveyObservationTabularDataContainer; diff --git a/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservation.tsx b/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservation.tsx index 97f598beec..5fad4c885c 100644 --- a/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservation.tsx +++ b/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservation.tsx @@ -1,9 +1,9 @@ import Box from '@mui/material/Box'; import { IStaticLayer, IStaticLayerFeature } from 'components/map/components/StaticLayers'; import { SURVEY_MAP_LAYER_COLOURS } from 'constants/colours'; +import SurveyObservationTabularDataContainer from 'features/surveys/view/components/data-container/SurveyObservationTabularDataContainer'; import { SurveySpatialMap } from 'features/surveys/view/survey-spatial/components/map/SurveySpatialMap'; import { SurveySpatialObservationPointPopup } from 'features/surveys/view/survey-spatial/components/observation/SurveySpatialObservationPointPopup'; -import { SurveySpatialObservationTable } from 'features/surveys/view/survey-spatial/components/observation/SurveySpatialObservationTable'; import SurveyMapTooltip from 'features/surveys/view/SurveyMapTooltip'; import { useBiohubApi } from 'hooks/useBioHubApi'; import { useSurveyContext } from 'hooks/useContext'; @@ -61,13 +61,13 @@ export const SurveySpatialObservation = () => { return ( <> {/* Display map with observation points */} - + {/* Display data table with observation details */} - - + + ); diff --git a/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservationTable.tsx b/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservationTable.tsx index db12d0ca4a..a47dc7181a 100644 --- a/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservationTable.tsx +++ b/app/src/features/surveys/view/survey-spatial/components/observation/SurveySpatialObservationTable.tsx @@ -45,7 +45,7 @@ export const SurveySpatialObservationTable = (props: ISurveyDataObservationTable const [totalRows, setTotalRows] = useState(0); const [page, setPage] = useState(0); - const [pageSize, setPageSize] = useState(5); + const [pageSize, setPageSize] = useState(10); const [sortModel, setSortModel] = useState([]); const [rows, setTableData] = useState([]); const [tableColumns, setTableColumns] = useState[]>([]); @@ -184,7 +184,7 @@ export const SurveySpatialObservationTable = (props: ISurveyDataObservationTable setPage(model.page); setPageSize(model.pageSize); }} - pageSizeOptions={[5]} + pageSizeOptions={[10, 25, 50]} paginationMode="server" sortingMode="server" sortModel={sortModel} @@ -193,6 +193,7 @@ export const SurveySpatialObservationTable = (props: ISurveyDataObservationTable getRowId={(row) => row.survey_observation_id} columns={tableColumns} rowSelection={false} + autoHeight={false} checkboxSelection={false} disableColumnSelector disableColumnFilter diff --git a/app/src/hooks/api/useAnalyticsApi.test.ts b/app/src/hooks/api/useAnalyticsApi.test.ts new file mode 100644 index 0000000000..cf7d72e18a --- /dev/null +++ b/app/src/hooks/api/useAnalyticsApi.test.ts @@ -0,0 +1,65 @@ +import axios from 'axios'; +import MockAdapter from 'axios-mock-adapter'; +import { IObservationCountByGroup } from 'interfaces/useAnalyticsApi.interface'; +import useAnalyticsApi from './useAnalyticsApi'; + +describe('useAnalyticsApi', () => { + let mock: MockAdapter; + + beforeEach(() => { + mock = new MockAdapter(axios); + }); + + afterEach(() => { + mock.restore(); + }); + + it('getObservationCountByGroup works as expected', async () => { + const response: IObservationCountByGroup[] = [ + { + id: '123-456-789', + row_count: 10, + individual_count: 40, + individual_percentage: 1, + itis_tsn: 123456, + observation_date: '2021-01-01', + survey_sample_site_id: 1, + survey_sample_method_id: 2, + survey_sample_period_id: 3, + qualitative_measurements: [ + { + taxon_measurement_id: '66', + measurement_name: 'a', + option: { + option_id: '1', + option_label: 'x' + } + } + ], + quantitative_measurements: [ + { + taxon_measurement_id: '77', + measurement_name: 'b', + value: 1 + } + ] + } + ]; + + mock.onGet('/api/analytics/observations').reply(200, response); + + const surveyIds = [1, 2]; + const groupByColumns = ['a', 'b']; + const groupByQuantitativeMeasurements = ['c', 'd']; + const groupByQualitativeMeasurements = ['e', 'f']; + + const result = await useAnalyticsApi(axios).getObservationCountByGroup( + surveyIds, + groupByColumns, + groupByQuantitativeMeasurements, + groupByQualitativeMeasurements + ); + + expect(result).toEqual(response); + }); +}); diff --git a/app/src/hooks/api/useAnalyticsApi.ts b/app/src/hooks/api/useAnalyticsApi.ts new file mode 100644 index 0000000000..b45ae447c3 --- /dev/null +++ b/app/src/hooks/api/useAnalyticsApi.ts @@ -0,0 +1,36 @@ +import { AxiosInstance } from 'axios'; +import { IObservationCountByGroup } from 'interfaces/useAnalyticsApi.interface'; + +/** + * Returns a set of supported api methods for working with survey analytics + * + * @param {AxiosInstance} axios + * @return {*} object whose properties are supported api methods. + */ +const useAnalyticsApi = (axios: AxiosInstance) => { + /** + * Create a new project survey + * + * @param {number[]} surveyIds + * @param {string[]} groupByColumns + * @param {string[]} groupByQualitativeMeasurements + * @param {string[]} groupByQuantitativeMeasurements + * @return {*} + */ + const getObservationCountByGroup = async ( + surveyIds: number[], + groupByColumns: string[], + groupByQuantitativeMeasurements: string[], + groupByQualitativeMeasurements: string[] + ): Promise => { + const { data } = await axios.get('/api/analytics/observations', { + params: { surveyIds, groupByColumns, groupByQuantitativeMeasurements, groupByQualitativeMeasurements } + }); + + return data; + }; + + return { getObservationCountByGroup }; +}; + +export default useAnalyticsApi; diff --git a/app/src/hooks/api/useObservationApi.ts b/app/src/hooks/api/useObservationApi.ts index 0ad602d329..525d6e483f 100644 --- a/app/src/hooks/api/useObservationApi.ts +++ b/app/src/hooks/api/useObservationApi.ts @@ -1,5 +1,9 @@ import { AxiosInstance, AxiosProgressEvent, CancelTokenSource } from 'axios'; import { IObservationsAdvancedFilters } from 'features/summary/tabular-data/observation/ObservationsListFilterForm'; +import { + CBQualitativeMeasurementTypeDefinition, + CBQuantitativeMeasurementTypeDefinition +} from 'interfaces/useCritterApi.interface'; import { IGetSurveyObservationsGeometryResponse, IGetSurveyObservationsResponse, @@ -137,6 +141,28 @@ const useObservationApi = (axios: AxiosInstance) => { return data; }; + /** + * Retrieves all measurements associated with all observation records + * + * @param {number} projectId + * @param {number} surveyId + * @return {*} {Promise} + */ + const getObservationMeasurementDefinitions = async ( + projectId: number, + surveyId: number + ): Promise<{ + qualitative_measurements: CBQualitativeMeasurementTypeDefinition[]; + quantitative_measurements: CBQuantitativeMeasurementTypeDefinition[]; + }> => { + const { data } = await axios.get<{ + qualitative_measurements: CBQualitativeMeasurementTypeDefinition[]; + quantitative_measurements: CBQuantitativeMeasurementTypeDefinition[]; + }>(`/api/project/${projectId}/survey/${surveyId}/observations/measurements`); + + return data; + }; + /** * Retrieves all survey observation records for the given survey * @@ -314,6 +340,7 @@ const useObservationApi = (axios: AxiosInstance) => { getObservedSpecies, findObservations, getObservationsGeometry, + getObservationMeasurementDefinitions, deleteObservationRecords, deleteObservationMeasurements, deleteObservationEnvironments, diff --git a/app/src/hooks/useBioHubApi.ts b/app/src/hooks/useBioHubApi.ts index 2443e19f5e..0115bd2df5 100644 --- a/app/src/hooks/useBioHubApi.ts +++ b/app/src/hooks/useBioHubApi.ts @@ -3,6 +3,7 @@ import useReferenceApi from 'hooks/api/useReferenceApi'; import { useConfigContext } from 'hooks/useContext'; import { useMemo } from 'react'; import useAdminApi from './api/useAdminApi'; +import useAnalyticsApi from './api/useAnalyticsApi'; import useAnimalApi from './api/useAnimalApi'; import useAxios from './api/useAxios'; import useCodesApi from './api/useCodesApi'; @@ -31,6 +32,8 @@ export const useBiohubApi = () => { const config = useConfigContext(); const apiAxios = useAxios(config.API_HOST); + const analytics = useAnalyticsApi(apiAxios); + const project = useProjectApi(apiAxios); const projectParticipants = useProjectParticipationApi(apiAxios); @@ -71,6 +74,7 @@ export const useBiohubApi = () => { return useMemo( () => ({ + analytics, project, projectParticipants, taxonomy, diff --git a/app/src/interfaces/useAnalyticsApi.interface.ts b/app/src/interfaces/useAnalyticsApi.interface.ts new file mode 100644 index 0000000000..1e15f26046 --- /dev/null +++ b/app/src/interfaces/useAnalyticsApi.interface.ts @@ -0,0 +1,31 @@ +interface IQualitativeMeasurementGroup { + taxon_measurement_id: string; + measurement_name: string; + option: { + option_id: string; + option_label: string; + }; +} + +interface IQuantitativeMeasurementGroup { + taxon_measurement_id: string; + measurement_name: string; + value: number; +} + +export interface IObservationCountByGroup { + /** + * Randomly generated unique ID for the group. + */ + id: string; + row_count: number; + individual_count: number; + individual_percentage: number; + itis_tsn?: number; + observation_date?: string; + survey_sample_site_id?: number; + survey_sample_method_id?: number; + survey_sample_period_id?: number; + qualitative_measurements: IQualitativeMeasurementGroup[]; + quantitative_measurements: IQuantitativeMeasurementGroup[]; +} diff --git a/app/src/interfaces/useSamplingSiteApi.interface.ts b/app/src/interfaces/useSamplingSiteApi.interface.ts index 675839738c..33799d4129 100644 --- a/app/src/interfaces/useSamplingSiteApi.interface.ts +++ b/app/src/interfaces/useSamplingSiteApi.interface.ts @@ -117,6 +117,7 @@ export interface IGetSampleMethodRecord { export interface IGetSampleMethodDetails extends IGetSampleMethodRecord { technique: { method_technique_id: number; + method_lookup_id: number; name: string; description: string; attractants: number[];