feat: updated telemetry import endpoint + added some todos for CSVConfig

api/src/paths/project/{projectId}/survey/{surveyId}/telemetry/import.ts

@@ -2,10 +2,12 @@ import { RequestHandler } from 'express';
 import { Operation } from 'express-openapi';
 import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles';
 import { getDBConnection } from '../../../../../../database/db';
+import { HTTP422CSVValidationError } from '../../../../../../errors/http-error';
 import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv';
 import { csvFileSchema } from '../../../../../../openapi/schemas/file';
 import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
 import { ImportTelemetryService } from '../../../../../../services/import-services/telemetry/import-telemetry-service';
+import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface';
 import { getLogger } from '../../../../../../utils/logger';
 import { parseMulterFile } from '../../../../../../utils/media/media-utils';
 import { getFileFromRequest } from '../../../../../../utils/request';
@@ -117,15 +119,21 @@ export function importTelemetryCSV(): RequestHandler {
 
       const telemetryService = new ImportTelemetryService(connection, worksheet, surveyId);
 
-      await telemetryService.importCSVWorksheet();
+      const errors = await telemetryService.importCSVWorksheet();
+
+      if (errors.length) {
+        throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors);
+      }
 
       await connection.commit();
 
       return res.status(200).send();
     } catch (error) {
-      defaultLog.error({ label: 'importTelemetry', message: 'error', error });
-      await connection.rollback();
+      if (error instanceof HTTP422CSVValidationError === false) {
+        defaultLog.error({ label: 'importTelemetry', message: 'error', error });
+      }
 
+      await connection.rollback();
       throw error;
     } finally {
       connection.release();

api/src/repositories/telemetry-repositories/telemetry-manual-repository.ts

@@ -38,22 +38,21 @@ export class TelemetryManualRepository extends BaseRepository {
    *
    * Note: Deployment IDs need to be pre-validated against the survey ID in the service.
    *
+   * TODO: Return a warning if the telemetry records count is less than the input telemetry count
+   *
    * @param {CreateManualTelemetry[]} telemetry - List of manual telemetry data to create
    * @returns {Promise<void>}
    */
   async bulkCreateManualTelemetry(telemetry: CreateManualTelemetry[]): Promise<void> {
     const knex = getKnex();
 
-    const queryBuilder = knex.insert(telemetry).into('telemetry_manual');
-
-    const response = await this.connection.knex(queryBuilder);
+    const queryBuilder = knex
+      .insert(telemetry)
+      .into('telemetry_manual')
+      .onConflict(['deployment_id', 'acquisition_date', 'latitude', 'longitude'])
+      .ignore();
 
-    if (response.rowCount !== telemetry.length) {
-      throw new ApiExecuteSQLError('Failed to create manual telemetry records', [
-        'TelemetryManualRepository->bulkCreateManualTelemetry',
-        `expected rowCount to be ${telemetry.length}, got ${response.rowCount}`
-      ]);
-    }
+    await this.connection.knex(queryBuilder);
   }
 
   /**

api/src/services/import-services/telemetry/import-telemetry-service.ts

@@ -1,12 +1,11 @@
 import { WorkSheet } from 'xlsx';
 import { z } from 'zod';
 import { IDBConnection } from '../../../database/db';
-import { HTTP422CSVValidationError } from '../../../errors/http-error';
 import { CodeRepository } from '../../../repositories/code-repository';
 import { CreateManualTelemetry } from '../../../repositories/telemetry-repositories/telemetry-manual-repository.interface';
 import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils';
 import { validateCSVWorksheet } from '../../../utils/csv-utils/csv-config-validation';
-import { CSVConfig, CSV_ERROR_MESSAGE } from '../../../utils/csv-utils/csv-config-validation.interface';
+import { CSVConfig, CSVError } from '../../../utils/csv-utils/csv-config-validation.interface';
 import { getTimeCellSetter, getTimeCellValidator, validateZodCell } from '../../../utils/csv-utils/csv-header-configs';
 import { getLogger } from '../../../utils/logger';
 import { DBService } from '../../db-service';
@@ -47,8 +46,8 @@ export class ImportTelemetryService extends DBService {
 
     const initialConfig: CSVConfig<TelemetryCSVStaticHeader> = {
       staticHeadersConfig: {
-        SERIAL: { aliases: ['DEVICE_ID'] },
-        VENDOR: { aliases: [] },
+        SERIAL: { aliases: ['DEVICE_ID', 'DEVICE ID', 'DEVICE', 'COLLAR', 'COLLAR ID'] },
+        VENDOR: { aliases: ['MAKE', 'MANUFACTURER'] },
         LATITUDE: { aliases: ['LAT'] },
         LONGITUDE: { aliases: ['LON', 'LONG', 'LNG'] },
         DATE: { aliases: [] },
@@ -72,15 +71,15 @@ export class ImportTelemetryService extends DBService {
    *
    * @async
    * @throws {ApiGeneralError} - If unable to fully insert records into SIMS
-   * @returns {*} {Promise<CSVError[]>}
+   * @returns {*} {Promise<CSVError[]>} List of CSV errors encountered during import
    */
-  async importCSVWorksheet(): Promise<void> {
+  async importCSVWorksheet(): Promise<CSVError[]> {
     const config = await this.getCSVConfig();
 
     const { errors, rows } = validateCSVWorksheet(this.worksheet, config);
 
     if (errors.length) {
-      throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors);
+      return errors;
     }
 
     const telemetry: CreateManualTelemetry[] = rows.map((row) => ({
@@ -93,11 +92,13 @@ export class ImportTelemetryService extends DBService {
 
     defaultLog.info({
       label: 'importCSVWorksheet',
-      message: 'Inserting telemetry records into SIMS',
+      message: 'Batch creating telemetry records',
       telemetryCount: telemetry.length
     });
 
     await this.telemetryVendorService.bulkCreateTelemetryInBatches(this.surveyId, telemetry);
+
+    return [];
   }
 
   /**

api/src/services/telemetry-services/telemetry-vendor-service.ts

@@ -250,16 +250,6 @@ export class TelemetryVendorService extends DBService {
     const batchSize = 500; // Max telemetry records to insert in a single query
     const concurrent = 10; // Max concurrent queries
 
-    const deploymentIds = [...new Set(telemetry.map((record) => record.deployment_id))];
-    const deployments = await this.deploymentService.getDeploymentsForSurvey(surveyId, deploymentIds);
-
-    if (deployments.length !== deploymentIds.length) {
-      throw new ApiGeneralError('Failed to bulk create manual telemetry', [
-        'TelemetryVendorService->bulkCreateManualTelemetryInBatches',
-        'survey missing reference to one or more deployment IDs'
-      ]);
-    }
-
     // Split the teletry into batches to prevent SQL cap error
     const telemetryBatches = chunk(telemetry, batchSize);
 

api/src/utils/csv-utils/csv-config-validation.interface.ts

@@ -4,6 +4,9 @@ export const CSV_ERROR_MESSAGE =
 /**
  * The CSV configuration interface
  *
+ * TODO:
+ *  1. Allow or disallow duplicate CSV rows
+ *  2. Support CSVWarnings
  */
 export interface CSVConfig<THeader extends Uppercase<string> = Uppercase<string>> {
   /**