feat: partially complete telemetry import state

api/src/services/import-services/telemetry/telemetry-header-configs.ts

@@ -1,6 +1,7 @@
 import { ExtendedDeploymentRecord } from '../../../repositories/telemetry-repositories/telemetry-deployment-repository.interface';
 import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils';
 import { CSVCellValidator } from '../../../utils/csv-utils/csv-config-validation.interface';
+import { setToLowercase } from '../../../utils/string-utils';
 import { getTelemetryDeviceKey } from '../../telemetry-services/telemetry-utils';
 import { TelemetryCSVStaticHeader } from './import-telemetry-service';
 
@@ -10,8 +11,10 @@ import { TelemetryCSVStaticHeader } from './import-telemetry-service';
  * @returns {*} {CSVCellValidator} The validate cell callback
  */
 export const getTelemetryVendorCellValidator = (vendors: Set<string>): CSVCellValidator => {
+  const vendorsLowerCased = setToLowercase(vendors);
+
   return (params) => {
-    if (vendors.has(String(params.cell).toLowerCase())) {
+    if (vendorsLowerCased.has(String(params.cell).toLowerCase())) {
       return [];
     }
 
@@ -44,12 +47,12 @@ export const getTelemetrySerialCellValidator = (
 
   // Populate the dictionary: device_key -> deployment
   for (const deployment of deployments) {
-    dictionary.set(deployment.device_key, deployment);
+    dictionary.set(deployment.device_key.toLowerCase(), deployment);
   }
 
   return (params) => {
     const serial = Number(params.cell);
-    const vendor = utils.getCellValue('VENDOR', params.row);
+    const vendor = String(utils.getCellValue('VENDOR', params.row)).toLowerCase();
     const deviceKey = getTelemetryDeviceKey({ vendor, serial });
     const deployment = dictionary.get(deviceKey);
 
@@ -65,21 +68,6 @@ export const getTelemetrySerialCellValidator = (
     // Mutate the cell to the deployment ID
     params.mutateCell = deployment.deployment_id;
 
-    // TODO: Keeping for when we support "CSVWarnings" in the CSV import
-    //
-    //const timestampInDeploymentRange =
-    //  timestamp >= deployment.attachment_start_timestamp &&
-    //  (deployment.attachment_end_timestamp === null || timestamp <= deployment.attachment_end_timestamp);
-    //
-    //if (!timestampInDeploymentRange) {
-    //  return [
-    //    {
-    //      error: `Timestamp not within deployment range`,
-    //      solution: `Check the telemetry timestamp is within the deployment range`
-    //    }
-    //  ];
-    //}
-
     return [];
   };
 };

api/src/services/telemetry-services/telemetry-vendor-service.ts

@@ -247,19 +247,21 @@ export class TelemetryVendorService extends DBService {
    * @returns {*} {Promise<void>}
    */
   async bulkCreateTelemetryInBatches(surveyId: number, telemetry: CreateManualTelemetry[]): Promise<void> {
-    const batchSize = 500; // Max telemetry records to insert in a single query
-    const concurrent = 10; // Max concurrent queries
+    // Max telemetry records to insert in a single query
+    const TELEMETRY_BATCH_SIZE = 500;
+    // Max concurrent queries
+    const CONCURRENT_QUERIES = 10;
 
     // Split the teletry into batches to prevent SQL cap error
-    const telemetryBatches = chunk(telemetry, batchSize);
+    const telemetryBatches = chunk(telemetry, TELEMETRY_BATCH_SIZE);
 
     // Create the async task processor
     const telemetryProcessor = async (telemetryBatch: CreateManualTelemetry[]): Promise<void> => {
       return this.bulkCreateManualTelemetry(surveyId, telemetryBatch);
     };
 
     // Process the telemetry in batches
-    const queueResult = await taskQueue(telemetryBatches, telemetryProcessor, concurrent);
+    const queueResult = await taskQueue(telemetryBatches, telemetryProcessor, CONCURRENT_QUERIES);
 
     // Check for any errors in the batch processing
     const batchErrors = queueResult.filter((result) => result.error);

api/src/utils/csv-utils/csv-config-validation.interface.ts

@@ -8,6 +8,7 @@ export const CSV_ERROR_MESSAGE =
  *  1. Allow or disallow duplicate CSV rows
  *    - Similar to a DB unique constraint? ie: ['NAME', 'AGE']
  *  2. Support CSVWarnings
+ *  3. Support CSVRowValidation? ie: Validate the entire row before / after the cell validation
  */
 export interface CSVConfig<THeader extends Uppercase<string> = Uppercase<string>> {
   /**

api/src/utils/csv-utils/csv-config-validation.ts

@@ -25,7 +25,7 @@ export const validateCSVWorksheet = <StaticHeaderType extends Uppercase<string>>
   const rows: CSVRowValidated<StaticHeaderType>[] = [];
   const errors = validateCSVHeaders(worksheet, config);
 
-  // If there are errors in the headers return early
+  // If there are errors in the headers, return early
   if (errors.length) {
     return { errors: errors, rows: [] };
   }

api/src/utils/csv-utils/csv-header-configs.ts

@@ -22,7 +22,7 @@ export const validateZodCell = (params: CSVParams, schema: z.ZodSchema, solution
     // Custom error message mapping
     errorMap: (_issue, ctx) => {
       if (ctx.defaultError === 'Required') {
-        return { message: 'Cell required' };
+        return { message: 'Cell is required' };
       }
 
       return { message: ctx.defaultError };

app/src/components/csv/CSVErrorsTable.tsx

@@ -83,7 +83,7 @@ export const CSVErrorsTable = (props: CSVErrorsTableProps) => {
       rows={rows}
       getRowId={(row) => row.id}
       columns={columns}
-      pageSizeOptions={[10, 25, 50]}
+      pageSizeOptions={[5, 10, 25, 50]}
       rowSelection={false}
       checkboxSelection={false}
       sortingOrder={['asc', 'desc']}

app/src/components/dialog/attachments/FileUploadSingleItemDialog.tsx

@@ -18,6 +18,7 @@ interface IFileUploadSingleItemDialog {
   uploadButtonLabel: string;
   onUpload: (file: File) => Promise<void>;
   onClose?: () => void;
+  onCancel?: () => void;
   dropZoneProps: Pick<IDropZoneConfigProps, 'acceptedFileExtensions' | 'maxFileSize'>;
 }
 

app/src/features/surveys/telemetry/table/TelemetryTableContainer.tsx

@@ -15,10 +15,12 @@ import Paper from '@mui/material/Paper';
 import Stack from '@mui/material/Stack';
 import Toolbar from '@mui/material/Toolbar';
 import Typography from '@mui/material/Typography';
+import axios, { AxiosProgressEvent } from 'axios';
 import { CSVErrorsTableContainer } from 'components/csv/CSVErrorsTableContainer';
 import DataGridValidationAlert from 'components/data-grid/DataGridValidationAlert';
-import { FileUploadSingleItemDialog } from 'components/dialog/attachments/FileUploadSingleItemDialog';
 import YesNoDialog from 'components/dialog/YesNoDialog';
+import { UploadFileStatus } from 'components/file-upload/FileUploadItem';
+import { FileUploadSingleItem } from 'components/file-upload/FileUploadSingleItem';
 import { TelemetryTableI18N } from 'constants/i18n';
 import { DialogContext, ISnackbarProps } from 'contexts/dialogContext';
 import { SurveyContext } from 'contexts/surveyContext';
@@ -28,7 +30,8 @@ import { APIError } from 'hooks/api/useAxios';
 import { useBiohubApi } from 'hooks/useBioHubApi';
 import { useTelemetryTableContext } from 'hooks/useContext';
 import { useContext, useDeferredValue, useState } from 'react';
-import { CSVError, isCSVValidationError } from 'utils/file-utils';
+import { CSVError, isCSVValidationError } from 'utils/csv-utils';
+import { getAxiosProgress } from 'utils/Utils';
 
 export const TelemetryTableContainer = () => {
   const biohubApi = useBiohubApi();
@@ -37,12 +40,21 @@ export const TelemetryTableContainer = () => {
   const telemetryTableContext = useTelemetryTableContext();
   const surveyContext = useContext(SurveyContext);
 
-  const [showImportDialog, setShowImportDialog] = useState(false);
   const [processingRecords, setProcessingRecords] = useState(false);
   const [showConfirmRemoveAllDialog, setShowConfirmRemoveAllDialog] = useState(false);
   const [contextMenuAnchorEl, setContextMenuAnchorEl] = useState<Element | null>(null);
   const [columnVisibilityMenuAnchorEl, setColumnVisibilityMenuAnchorEl] = useState<Element | null>(null);
+
+  // Telemetry import dialog state
+  const [showImportDialog, setShowImportDialog] = useState(false);
   const [importCSVErrors, setImportCSVErrors] = useState<CSVError[]>([]);
+  const [file, setFile] = useState<File | null>(null);
+  const [uploadStatus, setUploadStatus] = useState<UploadFileStatus>(UploadFileStatus.STAGED);
+  const [progress, setProgress] = useState<number>(0);
+
+  const isUploading = uploadStatus === UploadFileStatus.UPLOADING || uploadStatus === UploadFileStatus.FINISHING_UPLOAD;
+  const disableImportButton = isUploading || !file || uploadStatus === UploadFileStatus.FAILED;
+  const cancelToken = axios.CancelToken.source();
 
   const deferredUnsavedChanges = useDeferredValue(telemetryTableContext.hasUnsavedChanges);
 
@@ -60,14 +72,21 @@ export const TelemetryTableContainer = () => {
     setColumnVisibilityMenuAnchorEl(null);
   };
 
+  const handleResetFileImport = () => {
+    setFile(null);
+    setImportCSVErrors([]);
+    setUploadStatus(UploadFileStatus.STAGED);
+    setProgress(0);
+  };
+
   /**
    * Handle the close of the import dialog.
    *
    * @returns {*} {void}
    */
   const handleCloseImportDialog = () => {
-    setImportCSVErrors([]);
     setShowImportDialog(false);
+    handleResetFileImport();
   };
 
   /**
@@ -79,7 +98,20 @@ export const TelemetryTableContainer = () => {
    */
   const handleImportTelemetry = async (file: File) => {
     try {
-      await biohubApi.telemetry.importManualTelemetryCSV(surveyContext.projectId, surveyContext.surveyId, file);
+      setUploadStatus(UploadFileStatus.UPLOADING);
+
+      await biohubApi.telemetry.importManualTelemetryCSV(
+        surveyContext.projectId,
+        surveyContext.surveyId,
+        file,
+        cancelToken,
+        async (progressEvent: AxiosProgressEvent) => {
+          setProgress(getAxiosProgress(progressEvent));
+          if (progressEvent.loaded === progressEvent.total) {
+            setUploadStatus(UploadFileStatus.FINISHING_UPLOAD);
+          }
+        }
+      );
 
       setShowImportDialog(false);
 
@@ -96,7 +128,10 @@ export const TelemetryTableContainer = () => {
       telemetryTableContext.refreshRecords().then(() => {
         setProcessingRecords(false);
       });
+      setUploadStatus(UploadFileStatus.COMPLETE);
     } catch (error) {
+      setUploadStatus(UploadFileStatus.FAILED);
+
       if (isCSVValidationError(error)) {
         setImportCSVErrors(error.errors);
         return;
@@ -124,19 +159,41 @@ export const TelemetryTableContainer = () => {
 
   return (
     <>
-      <FileUploadSingleItemDialog
+      <YesNoDialog
+        dialogTitle={'Import Telemetry'}
+        dialogText={''}
         open={showImportDialog}
-        dialogTitle="Import Telemetry CSV"
+        yesButtonLabel={'Import'}
+        noButtonLabel={'Cancel'}
         onClose={handleCloseImportDialog}
-        onUpload={handleImportTelemetry}
-        uploadButtonLabel="Import"
-        dropZoneProps={{ acceptedFileExtensions: '.csv' }}>
-        {importCSVErrors.length > 0 ? (
-          <Box sx={{ mt: 2 }}>
-            <CSVErrorsTableContainer errors={importCSVErrors} />
-          </Box>
-        ) : null}
-      </FileUploadSingleItemDialog>
+        onNo={handleCloseImportDialog}
+        onYes={() => {
+          if (file) {
+            handleImportTelemetry(file);
+          }
+        }}
+        dialogContent={
+          <>
+            <FileUploadSingleItem
+              file={file}
+              status={uploadStatus}
+              progress={progress}
+              onFile={(file) => setFile(file)}
+              onCancel={handleResetFileImport}
+            />
+            {importCSVErrors.length > 0 ? (
+              <Box sx={{ mt: 2 }}>
+                <CSVErrorsTableContainer errors={importCSVErrors} />
+              </Box>
+            ) : null}
+          </>
+        }
+        yesButtonProps={{
+          loading: isUploading,
+          disabled: disableImportButton
+        }}
+      />
+
       <YesNoDialog
         dialogTitle={TelemetryTableI18N.removeAllDialogTitle}
         dialogText={TelemetryTableI18N.removeAllDialogText}
@@ -173,8 +230,6 @@ export const TelemetryTableContainer = () => {
               variant="contained"
               color="primary"
               startIcon={<Icon path={mdiImport} size={1} />}
-              //// TODO: Disabled while the backend CSV Import code is being refactored (https://apps.nrs.gov.bc.ca/int/jira/browse/SIMSBIOHUB-652)
-              //disabled={true}
               onClick={() => setShowImportDialog(true)}>
               Import
             </Button>