From b0a9d4b4ea1c00d9d0b417993d9b04638533f312 Mon Sep 17 00:00:00 2001 From: Mac Deluca Date: Fri, 20 Dec 2024 15:58:26 -0800 Subject: [PATCH] feat: partially complete telemetry import state --- .../telemetry/telemetry-header-configs.ts | 24 ++--- .../telemetry-vendor-service.ts | 10 +- .../csv-config-validation.interface.ts | 1 + .../utils/csv-utils/csv-config-validation.ts | 2 +- api/src/utils/csv-utils/csv-header-configs.ts | 2 +- app/src/components/csv/CSVErrorsTable.tsx | 2 +- .../FileUploadSingleItemDialog.tsx | 1 + .../table/TelemetryTableContainer.tsx | 91 +++++++++++++++---- 8 files changed, 90 insertions(+), 43 deletions(-) diff --git a/api/src/services/import-services/telemetry/telemetry-header-configs.ts b/api/src/services/import-services/telemetry/telemetry-header-configs.ts index 906f5ced3d..aad66c8f03 100644 --- a/api/src/services/import-services/telemetry/telemetry-header-configs.ts +++ b/api/src/services/import-services/telemetry/telemetry-header-configs.ts @@ -1,6 +1,7 @@ import { ExtendedDeploymentRecord } from '../../../repositories/telemetry-repositories/telemetry-deployment-repository.interface'; import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; import { CSVCellValidator } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { setToLowercase } from '../../../utils/string-utils'; import { getTelemetryDeviceKey } from '../../telemetry-services/telemetry-utils'; import { TelemetryCSVStaticHeader } from './import-telemetry-service'; @@ -10,8 +11,10 @@ import { TelemetryCSVStaticHeader } from './import-telemetry-service'; * @returns {*} {CSVCellValidator} The validate cell callback */ export const getTelemetryVendorCellValidator = (vendors: Set): CSVCellValidator => { + const vendorsLowerCased = setToLowercase(vendors); + return (params) => { - if (vendors.has(String(params.cell).toLowerCase())) { + if (vendorsLowerCased.has(String(params.cell).toLowerCase())) { return []; } @@ -44,12 +47,12 @@ export const getTelemetrySerialCellValidator = ( // Populate the dictionary: device_key -> deployment for (const deployment of deployments) { - dictionary.set(deployment.device_key, deployment); + dictionary.set(deployment.device_key.toLowerCase(), deployment); } return (params) => { const serial = Number(params.cell); - const vendor = utils.getCellValue('VENDOR', params.row); + const vendor = String(utils.getCellValue('VENDOR', params.row)).toLowerCase(); const deviceKey = getTelemetryDeviceKey({ vendor, serial }); const deployment = dictionary.get(deviceKey); @@ -65,21 +68,6 @@ export const getTelemetrySerialCellValidator = ( // Mutate the cell to the deployment ID params.mutateCell = deployment.deployment_id; - // TODO: Keeping for when we support "CSVWarnings" in the CSV import - // - //const timestampInDeploymentRange = - // timestamp >= deployment.attachment_start_timestamp && - // (deployment.attachment_end_timestamp === null || timestamp <= deployment.attachment_end_timestamp); - // - //if (!timestampInDeploymentRange) { - // return [ - // { - // error: `Timestamp not within deployment range`, - // solution: `Check the telemetry timestamp is within the deployment range` - // } - // ]; - //} - return []; }; }; diff --git a/api/src/services/telemetry-services/telemetry-vendor-service.ts b/api/src/services/telemetry-services/telemetry-vendor-service.ts index b0490fc231..c8a3f91cbc 100644 --- a/api/src/services/telemetry-services/telemetry-vendor-service.ts +++ b/api/src/services/telemetry-services/telemetry-vendor-service.ts @@ -247,11 +247,13 @@ export class TelemetryVendorService extends DBService { * @returns {*} {Promise} */ async bulkCreateTelemetryInBatches(surveyId: number, telemetry: CreateManualTelemetry[]): Promise { - const batchSize = 500; // Max telemetry records to insert in a single query - const concurrent = 10; // Max concurrent queries + // Max telemetry records to insert in a single query + const TELEMETRY_BATCH_SIZE = 500; + // Max concurrent queries + const CONCURRENT_QUERIES = 10; // Split the teletry into batches to prevent SQL cap error - const telemetryBatches = chunk(telemetry, batchSize); + const telemetryBatches = chunk(telemetry, TELEMETRY_BATCH_SIZE); // Create the async task processor const telemetryProcessor = async (telemetryBatch: CreateManualTelemetry[]): Promise => { @@ -259,7 +261,7 @@ export class TelemetryVendorService extends DBService { }; // Process the telemetry in batches - const queueResult = await taskQueue(telemetryBatches, telemetryProcessor, concurrent); + const queueResult = await taskQueue(telemetryBatches, telemetryProcessor, CONCURRENT_QUERIES); // Check for any errors in the batch processing const batchErrors = queueResult.filter((result) => result.error); diff --git a/api/src/utils/csv-utils/csv-config-validation.interface.ts b/api/src/utils/csv-utils/csv-config-validation.interface.ts index 86ec30a528..5c21b17546 100644 --- a/api/src/utils/csv-utils/csv-config-validation.interface.ts +++ b/api/src/utils/csv-utils/csv-config-validation.interface.ts @@ -8,6 +8,7 @@ export const CSV_ERROR_MESSAGE = * 1. Allow or disallow duplicate CSV rows * - Similar to a DB unique constraint? ie: ['NAME', 'AGE'] * 2. Support CSVWarnings + * 3. Support CSVRowValidation? ie: Validate the entire row before / after the cell validation */ export interface CSVConfig = Uppercase> { /** diff --git a/api/src/utils/csv-utils/csv-config-validation.ts b/api/src/utils/csv-utils/csv-config-validation.ts index 95a7bbccb6..7d4bb50b5e 100644 --- a/api/src/utils/csv-utils/csv-config-validation.ts +++ b/api/src/utils/csv-utils/csv-config-validation.ts @@ -25,7 +25,7 @@ export const validateCSVWorksheet = > const rows: CSVRowValidated[] = []; const errors = validateCSVHeaders(worksheet, config); - // If there are errors in the headers return early + // If there are errors in the headers, return early if (errors.length) { return { errors: errors, rows: [] }; } diff --git a/api/src/utils/csv-utils/csv-header-configs.ts b/api/src/utils/csv-utils/csv-header-configs.ts index 5045e2c43a..cf170ac64e 100644 --- a/api/src/utils/csv-utils/csv-header-configs.ts +++ b/api/src/utils/csv-utils/csv-header-configs.ts @@ -22,7 +22,7 @@ export const validateZodCell = (params: CSVParams, schema: z.ZodSchema, solution // Custom error message mapping errorMap: (_issue, ctx) => { if (ctx.defaultError === 'Required') { - return { message: 'Cell required' }; + return { message: 'Cell is required' }; } return { message: ctx.defaultError }; diff --git a/app/src/components/csv/CSVErrorsTable.tsx b/app/src/components/csv/CSVErrorsTable.tsx index bf97fc3c05..9ff5cff794 100644 --- a/app/src/components/csv/CSVErrorsTable.tsx +++ b/app/src/components/csv/CSVErrorsTable.tsx @@ -83,7 +83,7 @@ export const CSVErrorsTable = (props: CSVErrorsTableProps) => { rows={rows} getRowId={(row) => row.id} columns={columns} - pageSizeOptions={[10, 25, 50]} + pageSizeOptions={[5, 10, 25, 50]} rowSelection={false} checkboxSelection={false} sortingOrder={['asc', 'desc']} diff --git a/app/src/components/dialog/attachments/FileUploadSingleItemDialog.tsx b/app/src/components/dialog/attachments/FileUploadSingleItemDialog.tsx index a7ab1fadda..175b19d1f4 100644 --- a/app/src/components/dialog/attachments/FileUploadSingleItemDialog.tsx +++ b/app/src/components/dialog/attachments/FileUploadSingleItemDialog.tsx @@ -18,6 +18,7 @@ interface IFileUploadSingleItemDialog { uploadButtonLabel: string; onUpload: (file: File) => Promise; onClose?: () => void; + onCancel?: () => void; dropZoneProps: Pick; } diff --git a/app/src/features/surveys/telemetry/table/TelemetryTableContainer.tsx b/app/src/features/surveys/telemetry/table/TelemetryTableContainer.tsx index a8602abd21..1989cc7286 100644 --- a/app/src/features/surveys/telemetry/table/TelemetryTableContainer.tsx +++ b/app/src/features/surveys/telemetry/table/TelemetryTableContainer.tsx @@ -15,10 +15,12 @@ import Paper from '@mui/material/Paper'; import Stack from '@mui/material/Stack'; import Toolbar from '@mui/material/Toolbar'; import Typography from '@mui/material/Typography'; +import axios, { AxiosProgressEvent } from 'axios'; import { CSVErrorsTableContainer } from 'components/csv/CSVErrorsTableContainer'; import DataGridValidationAlert from 'components/data-grid/DataGridValidationAlert'; -import { FileUploadSingleItemDialog } from 'components/dialog/attachments/FileUploadSingleItemDialog'; import YesNoDialog from 'components/dialog/YesNoDialog'; +import { UploadFileStatus } from 'components/file-upload/FileUploadItem'; +import { FileUploadSingleItem } from 'components/file-upload/FileUploadSingleItem'; import { TelemetryTableI18N } from 'constants/i18n'; import { DialogContext, ISnackbarProps } from 'contexts/dialogContext'; import { SurveyContext } from 'contexts/surveyContext'; @@ -28,7 +30,8 @@ import { APIError } from 'hooks/api/useAxios'; import { useBiohubApi } from 'hooks/useBioHubApi'; import { useTelemetryTableContext } from 'hooks/useContext'; import { useContext, useDeferredValue, useState } from 'react'; -import { CSVError, isCSVValidationError } from 'utils/file-utils'; +import { CSVError, isCSVValidationError } from 'utils/csv-utils'; +import { getAxiosProgress } from 'utils/Utils'; export const TelemetryTableContainer = () => { const biohubApi = useBiohubApi(); @@ -37,12 +40,21 @@ export const TelemetryTableContainer = () => { const telemetryTableContext = useTelemetryTableContext(); const surveyContext = useContext(SurveyContext); - const [showImportDialog, setShowImportDialog] = useState(false); const [processingRecords, setProcessingRecords] = useState(false); const [showConfirmRemoveAllDialog, setShowConfirmRemoveAllDialog] = useState(false); const [contextMenuAnchorEl, setContextMenuAnchorEl] = useState(null); const [columnVisibilityMenuAnchorEl, setColumnVisibilityMenuAnchorEl] = useState(null); + + // Telemetry import dialog state + const [showImportDialog, setShowImportDialog] = useState(false); const [importCSVErrors, setImportCSVErrors] = useState([]); + const [file, setFile] = useState(null); + const [uploadStatus, setUploadStatus] = useState(UploadFileStatus.STAGED); + const [progress, setProgress] = useState(0); + + const isUploading = uploadStatus === UploadFileStatus.UPLOADING || uploadStatus === UploadFileStatus.FINISHING_UPLOAD; + const disableImportButton = isUploading || !file || uploadStatus === UploadFileStatus.FAILED; + const cancelToken = axios.CancelToken.source(); const deferredUnsavedChanges = useDeferredValue(telemetryTableContext.hasUnsavedChanges); @@ -60,14 +72,21 @@ export const TelemetryTableContainer = () => { setColumnVisibilityMenuAnchorEl(null); }; + const handleResetFileImport = () => { + setFile(null); + setImportCSVErrors([]); + setUploadStatus(UploadFileStatus.STAGED); + setProgress(0); + }; + /** * Handle the close of the import dialog. * * @returns {*} {void} */ const handleCloseImportDialog = () => { - setImportCSVErrors([]); setShowImportDialog(false); + handleResetFileImport(); }; /** @@ -79,7 +98,20 @@ export const TelemetryTableContainer = () => { */ const handleImportTelemetry = async (file: File) => { try { - await biohubApi.telemetry.importManualTelemetryCSV(surveyContext.projectId, surveyContext.surveyId, file); + setUploadStatus(UploadFileStatus.UPLOADING); + + await biohubApi.telemetry.importManualTelemetryCSV( + surveyContext.projectId, + surveyContext.surveyId, + file, + cancelToken, + async (progressEvent: AxiosProgressEvent) => { + setProgress(getAxiosProgress(progressEvent)); + if (progressEvent.loaded === progressEvent.total) { + setUploadStatus(UploadFileStatus.FINISHING_UPLOAD); + } + } + ); setShowImportDialog(false); @@ -96,7 +128,10 @@ export const TelemetryTableContainer = () => { telemetryTableContext.refreshRecords().then(() => { setProcessingRecords(false); }); + setUploadStatus(UploadFileStatus.COMPLETE); } catch (error) { + setUploadStatus(UploadFileStatus.FAILED); + if (isCSVValidationError(error)) { setImportCSVErrors(error.errors); return; @@ -124,19 +159,41 @@ export const TelemetryTableContainer = () => { return ( <> - - {importCSVErrors.length > 0 ? ( - - - - ) : null} - + onNo={handleCloseImportDialog} + onYes={() => { + if (file) { + handleImportTelemetry(file); + } + }} + dialogContent={ + <> + setFile(file)} + onCancel={handleResetFileImport} + /> + {importCSVErrors.length > 0 ? ( + + + + ) : null} + + } + yesButtonProps={{ + loading: isUploading, + disabled: disableImportButton + }} + /> + { variant="contained" color="primary" startIcon={} - //// TODO: Disabled while the backend CSV Import code is being refactored (https://apps.nrs.gov.bc.ca/int/jira/browse/SIMSBIOHUB-652) - //disabled={true} onClick={() => setShowImportDialog(true)}> Import