diff --git a/api/src/paths/project/{projectId}/survey/{surveyId}/telemetry/import.ts b/api/src/paths/project/{projectId}/survey/{surveyId}/telemetry/import.ts index f999736f17..0bd336384c 100644 --- a/api/src/paths/project/{projectId}/survey/{surveyId}/telemetry/import.ts +++ b/api/src/paths/project/{projectId}/survey/{surveyId}/telemetry/import.ts @@ -2,10 +2,12 @@ import { RequestHandler } from 'express'; import { Operation } from 'express-openapi'; import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../constants/roles'; import { getDBConnection } from '../../../../../../database/db'; +import { HTTP422CSVValidationError } from '../../../../../../errors/http-error'; import { CSVValidationErrorResponse } from '../../../../../../openapi/schemas/csv'; import { csvFileSchema } from '../../../../../../openapi/schemas/file'; import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization'; import { ImportTelemetryService } from '../../../../../../services/import-services/telemetry/import-telemetry-service'; +import { CSV_ERROR_MESSAGE } from '../../../../../../utils/csv-utils/csv-config-validation.interface'; import { getLogger } from '../../../../../../utils/logger'; import { parseMulterFile } from '../../../../../../utils/media/media-utils'; import { getFileFromRequest } from '../../../../../../utils/request'; @@ -117,15 +119,21 @@ export function importTelemetryCSV(): RequestHandler { const telemetryService = new ImportTelemetryService(connection, worksheet, surveyId); - await telemetryService.importCSVWorksheet(); + const errors = await telemetryService.importCSVWorksheet(); + + if (errors.length) { + throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors); + } await connection.commit(); return res.status(200).send(); } catch (error) { - defaultLog.error({ label: 'importTelemetry', message: 'error', error }); - await connection.rollback(); + if (error instanceof HTTP422CSVValidationError === false) { + defaultLog.error({ label: 'importTelemetry', message: 'error', error }); + } + await connection.rollback(); throw error; } finally { connection.release(); diff --git a/api/src/repositories/telemetry-repositories/telemetry-manual-repository.ts b/api/src/repositories/telemetry-repositories/telemetry-manual-repository.ts index 1302d6e070..19dbd8e0ee 100644 --- a/api/src/repositories/telemetry-repositories/telemetry-manual-repository.ts +++ b/api/src/repositories/telemetry-repositories/telemetry-manual-repository.ts @@ -38,22 +38,21 @@ export class TelemetryManualRepository extends BaseRepository { * * Note: Deployment IDs need to be pre-validated against the survey ID in the service. * + * TODO: Return a warning if the telemetry records count is less than the input telemetry count + * * @param {CreateManualTelemetry[]} telemetry - List of manual telemetry data to create * @returns {Promise} */ async bulkCreateManualTelemetry(telemetry: CreateManualTelemetry[]): Promise { const knex = getKnex(); - const queryBuilder = knex.insert(telemetry).into('telemetry_manual'); - - const response = await this.connection.knex(queryBuilder); + const queryBuilder = knex + .insert(telemetry) + .into('telemetry_manual') + .onConflict(['deployment_id', 'acquisition_date', 'latitude', 'longitude']) + .ignore(); - if (response.rowCount !== telemetry.length) { - throw new ApiExecuteSQLError('Failed to create manual telemetry records', [ - 'TelemetryManualRepository->bulkCreateManualTelemetry', - `expected rowCount to be ${telemetry.length}, got ${response.rowCount}` - ]); - } + await this.connection.knex(queryBuilder); } /** diff --git a/api/src/services/import-services/telemetry/import-telemetry-service.ts b/api/src/services/import-services/telemetry/import-telemetry-service.ts index e576eeb637..47f64a9489 100644 --- a/api/src/services/import-services/telemetry/import-telemetry-service.ts +++ b/api/src/services/import-services/telemetry/import-telemetry-service.ts @@ -1,12 +1,11 @@ import { WorkSheet } from 'xlsx'; import { z } from 'zod'; import { IDBConnection } from '../../../database/db'; -import { HTTP422CSVValidationError } from '../../../errors/http-error'; import { CodeRepository } from '../../../repositories/code-repository'; import { CreateManualTelemetry } from '../../../repositories/telemetry-repositories/telemetry-manual-repository.interface'; import { CSVConfigUtils } from '../../../utils/csv-utils/csv-config-utils'; import { validateCSVWorksheet } from '../../../utils/csv-utils/csv-config-validation'; -import { CSVConfig, CSV_ERROR_MESSAGE } from '../../../utils/csv-utils/csv-config-validation.interface'; +import { CSVConfig, CSVError } from '../../../utils/csv-utils/csv-config-validation.interface'; import { getTimeCellSetter, getTimeCellValidator, validateZodCell } from '../../../utils/csv-utils/csv-header-configs'; import { getLogger } from '../../../utils/logger'; import { DBService } from '../../db-service'; @@ -47,8 +46,8 @@ export class ImportTelemetryService extends DBService { const initialConfig: CSVConfig = { staticHeadersConfig: { - SERIAL: { aliases: ['DEVICE_ID'] }, - VENDOR: { aliases: [] }, + SERIAL: { aliases: ['DEVICE_ID', 'DEVICE ID', 'DEVICE', 'COLLAR', 'COLLAR ID'] }, + VENDOR: { aliases: ['MAKE', 'MANUFACTURER'] }, LATITUDE: { aliases: ['LAT'] }, LONGITUDE: { aliases: ['LON', 'LONG', 'LNG'] }, DATE: { aliases: [] }, @@ -72,15 +71,15 @@ export class ImportTelemetryService extends DBService { * * @async * @throws {ApiGeneralError} - If unable to fully insert records into SIMS - * @returns {*} {Promise} + * @returns {*} {Promise} List of CSV errors encountered during import */ - async importCSVWorksheet(): Promise { + async importCSVWorksheet(): Promise { const config = await this.getCSVConfig(); const { errors, rows } = validateCSVWorksheet(this.worksheet, config); if (errors.length) { - throw new HTTP422CSVValidationError(CSV_ERROR_MESSAGE, errors); + return errors; } const telemetry: CreateManualTelemetry[] = rows.map((row) => ({ @@ -93,11 +92,13 @@ export class ImportTelemetryService extends DBService { defaultLog.info({ label: 'importCSVWorksheet', - message: 'Inserting telemetry records into SIMS', + message: 'Batch creating telemetry records', telemetryCount: telemetry.length }); await this.telemetryVendorService.bulkCreateTelemetryInBatches(this.surveyId, telemetry); + + return []; } /** diff --git a/api/src/services/telemetry-services/telemetry-vendor-service.ts b/api/src/services/telemetry-services/telemetry-vendor-service.ts index e3f452d162..b0490fc231 100644 --- a/api/src/services/telemetry-services/telemetry-vendor-service.ts +++ b/api/src/services/telemetry-services/telemetry-vendor-service.ts @@ -250,16 +250,6 @@ export class TelemetryVendorService extends DBService { const batchSize = 500; // Max telemetry records to insert in a single query const concurrent = 10; // Max concurrent queries - const deploymentIds = [...new Set(telemetry.map((record) => record.deployment_id))]; - const deployments = await this.deploymentService.getDeploymentsForSurvey(surveyId, deploymentIds); - - if (deployments.length !== deploymentIds.length) { - throw new ApiGeneralError('Failed to bulk create manual telemetry', [ - 'TelemetryVendorService->bulkCreateManualTelemetryInBatches', - 'survey missing reference to one or more deployment IDs' - ]); - } - // Split the teletry into batches to prevent SQL cap error const telemetryBatches = chunk(telemetry, batchSize); diff --git a/api/src/utils/csv-utils/csv-config-validation.interface.ts b/api/src/utils/csv-utils/csv-config-validation.interface.ts index 25a412ba64..9eb96b0cae 100644 --- a/api/src/utils/csv-utils/csv-config-validation.interface.ts +++ b/api/src/utils/csv-utils/csv-config-validation.interface.ts @@ -4,6 +4,9 @@ export const CSV_ERROR_MESSAGE = /** * The CSV configuration interface * + * TODO: + * 1. Allow or disallow duplicate CSV rows + * 2. Support CSVWarnings */ export interface CSVConfig = Uppercase> { /**