diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 6ea93ad..d69036c 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -296,3 +296,21 @@ model aqi_units_xref { update_user_id String @db.VarChar(200) update_utc_timestamp DateTime @db.Timestamp(6) } + +model aqi_tissue_types { + aqi_tissue_types_id String @id @db.Uuid + custom_id String @db.VarChar(200) + create_user_id String @db.VarChar(200) + create_utc_timestamp DateTime @db.Timestamp(6) + update_user_id String @db.VarChar(200) + update_utc_timestamp DateTime @db.Timestamp(6) +} + +model aqi_sampling_agency { + aqi_sampling_agency_id String @id @db.Uuid + custom_id String @db.VarChar(200) + create_user_id String @db.VarChar(200) + create_utc_timestamp DateTime @db.Timestamp(6) + update_user_id String @db.VarChar(200) + update_utc_timestamp DateTime @db.Timestamp(6) +} diff --git a/backend/src/aqi_api/aqi_api.service.ts b/backend/src/aqi_api/aqi_api.service.ts index f13bf86..37005f3 100644 --- a/backend/src/aqi_api/aqi_api.service.ts +++ b/backend/src/aqi_api/aqi_api.service.ts @@ -31,7 +31,6 @@ export class AqiApiService { ); return response.data.id; } catch (err) { - console.log(body); this.logger.error( "API CALL TO POST Field Visits failed: ", err.response.data.message, @@ -510,12 +509,13 @@ export class AqiApiService { } mergeErrorMessages(localErrors: any[], remoteErrors: any[]) { - const map = new Map(); + const map = new Map(); const mergeItem = (item: any) => { - const exists = map.get(item.rowNum); + const key = `${item.rowNum}-${item.type}`; + const exists = map.get(key); map.set( - item.rowNum, + key, exists ? { ...exists, message: { ...exists.message, ...item.message } } : item, @@ -570,36 +570,38 @@ export class AqiApiService { } } - async deleteRelatedData(fileName: string) { - const guidsToDelete: any = await this.prisma.aqi_imported_data.findMany({ - where: { - file_name: fileName, - }, - }); - - console.log(guidsToDelete) + async ObservationDelete(obsData: any[]) { + if (obsData.length > 0) { + const batchSize = 50; + const observationBatches = []; + for (let i = 0; i < obsData.length; i += batchSize) { + observationBatches.push(obsData.slice(i, i + batchSize)); + } - // Delete all the observations from the list of imported guids - if (guidsToDelete[0].imported_guids.observations.length > 0) { - try { - let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v2/observations?ids=${guidsToDelete[0].imported_guids.observations}`, - { - headers: { - Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, - "x-api-key": process.env.AQI_ACCESS_TOKEN, + observationBatches.forEach(async (batch) => { + try { + let deletion = await axios.delete( + `${process.env.AQI_BASE_URL}/v2/observations?ids=${batch}`, + { + headers: { + Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, + "x-api-key": process.env.AQI_ACCESS_TOKEN, + }, }, - }, - ); - this.logger.log("AQI OBS DELETION: " + deletion.data); - } catch (err) { - this.logger.error(`API call to delete AQI observation failed: `, err); - } + ); + this.logger.log("AQI OBS DELETION: " + deletion.status); + } catch (err) { + this.logger.error(`API call to delete AQI observation failed: `, err); + } + }); } - // Delete all the specimens for the activities imported from AQI and the PSQL db - if (guidsToDelete[0].imported_guids.specimens.length > 0) { - for (const specimen of guidsToDelete[0].imported_guids.specimens) { + return new Promise((resolve) => setTimeout(resolve, 1000)); + } + + async SpecimenDelete(specimenData: any[]) { + if (specimenData.length > 0) { + for (const specimen of specimenData) { try { let aqiDeletion = await axios.delete( `${process.env.AQI_BASE_URL}/v1/specimens/${specimen}`, @@ -620,17 +622,25 @@ export class AqiApiService { }); this.logger.log("DB SPECIMEN DELETION: " + dbDeletion); } catch (err) { - this.logger.error(`API call to delete DB specimen failed: `, err); + if (err.code === "P2025") { + this.logger.log( + `Record with ID ${specimen} not found in DB. Record was deleted in AQI but skipping deletion from DB.`, + ); + } else { + this.logger.error(`API call to delete DB specimen failed: `, err); + } } } catch (err) { this.logger.error(`API call to delete AQI specimen failed: `, err); } } } + return new Promise((resolve) => setTimeout(resolve, 1000)); + } - // Delete all the activities for the visits imported - if (guidsToDelete[0].imported_guids.activities.length > 0) { - for (const activity of guidsToDelete[0].imported_guids.activities) { + async ActivityDelete(activityData: any[]) { + if (activityData.length > 0) { + for (const activity of activityData) { try { let aqiDeletion = await axios.delete( `${process.env.AQI_BASE_URL}/v1/activities/${activity}`, @@ -651,19 +661,27 @@ export class AqiApiService { }); this.logger.log("DB ACTIVITY DELETION: " + dbDeletion); } catch (err) { - this.logger.error(`API call to delete DB activity failed: `, err); + if (err.code === "P2025") { + this.logger.log( + `Record with ID ${activity} not found in DB. Record was deleted in AQI but skipping deletion from DB.`, + ); + } else { + this.logger.error(`API call to delete DB activity failed: `, err); + } } } catch (err) { this.logger.error(`API call to delete AQI activity failed: `, err); } } } + return new Promise((resolve) => setTimeout(resolve, 1000)); + } - // Delete all the visits for the visits imported - if (guidsToDelete[0].imported_guids.visits.length > 0) { + async VisitDelete(visitData: any[]) { + if (visitData.length > 0) { try { let deletion = await axios.delete( - `${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${guidsToDelete[0].imported_guids.visits}`, + `${process.env.AQI_BASE_URL}/v1/fieldvisits?ids=${visitData}`, { headers: { Authorization: `token ${process.env.AQI_ACCESS_TOKEN}`, @@ -677,23 +695,92 @@ export class AqiApiService { const dbDeletion = await this.prisma.aqi_field_visits.deleteMany({ where: { aqi_field_visits_id: { - in: guidsToDelete[0].imported_guids.visits, + in: visitData, }, }, }); this.logger.log("DB VISIT DELETION: " + dbDeletion); } catch (err) { - this.logger.error(`API call to delete DB visits failed: `, err); + if (err.code === "P2025") { + this.logger.log( + `Records with IDs ${visitData} not found in DB. Records were deleted in AQI but skipping deletion from DB.`, + ); + } else { + this.logger.error(`API call to delete DB visits failed: `, err); + } } } catch (err) { this.logger.error(`API call to delete AQI visit failed: `, err); } } + return new Promise((resolve) => setTimeout(resolve, 1000)); + } - await this.prisma.aqi_imported_data.deleteMany({ + async deleteRelatedData(fileName: string) { + const guidsToDelete: any = await this.prisma.aqi_imported_data.findMany({ where: { file_name: fileName, }, }); + + let successfulObs = false + let successfulSpecimen = false + let successfulActivity = false + let successfulVisit = false + + // Delete all the observations in AQI that are in the list of imported guids + this.logger.log( + `Starting observation delete for file ${fileName}..............`, + ); + + await this.ObservationDelete( + guidsToDelete[0].imported_guids.observations, + ).then(() => { + successfulObs = true + this.logger.log(`Finished observation delete for file ${fileName}`) + }) + + if (successfulObs){ + // Delete all the specimens that were imported for the file from AQI and the PSQL db + this.logger.log( + `Starting specimen delete for file ${fileName}..............`, + ); + await this.SpecimenDelete(guidsToDelete[0].imported_guids.specimens).then( + () => { + successfulSpecimen = true; + this.logger.log(`Finished specimen delete for file ${fileName}.`); + }, + ); + } + + if (successfulSpecimen){ + // Delete all the activities for the visits imported + this.logger.log( + `Starting activity delete for file ${fileName}..............`, + ); + await this.ActivityDelete(guidsToDelete[0].imported_guids.activities).then(() => { + successfulActivity = true; + this.logger.log(`Finished activity delete for file ${fileName}.`); + }); + } + + if (successfulActivity){ + // Delete all the visits for the visits imported + this.logger.log(`Starting visit delete for file ${fileName}..............`); + await this.VisitDelete(guidsToDelete[0].imported_guids.visits).then(() => { + successfulVisit = true + this.logger.log(`Finished visit delete for file ${fileName}.`); + }); + } + + if (successfulObs && successfulSpecimen && successfulActivity && successfulVisit){ + await this.prisma.aqi_imported_data.deleteMany({ + where: { + file_name: fileName, + }, + }); + }else{ + this.logger.error(`Error deleting related data for file ${fileName}.`); + } } } diff --git a/backend/src/cron-job/cron-job.service.ts b/backend/src/cron-job/cron-job.service.ts index f114501..d2da5f0 100644 --- a/backend/src/cron-job/cron-job.service.ts +++ b/backend/src/cron-job/cron-job.service.ts @@ -15,6 +15,7 @@ export class CronJobService { private readonly logger = new Logger(CronJobService.name); private tableModels; + private isProcessing = false; private dataPullDownComplete: boolean = false; constructor( @@ -34,6 +35,8 @@ export class CronJobService { ["aqi_detection_conditions", this.prisma.aqi_detection_conditions], ["aqi_result_status", this.prisma.aqi_result_status], ["aqi_result_grade", this.prisma.aqi_result_grade], + ["aqi_tissue_types", this.prisma.aqi_tissue_types], + ["aqi_sampling_agency", this.prisma.aqi_sampling_agency], ["aqi_locations", this.prisma.aqi_locations], ["aqi_field_visits", this.prisma.aqi_field_visits], ["aqi_field_activities", this.prisma.aqi_field_activities], @@ -108,6 +111,19 @@ export class CronJobService { dbTable: "aqi_result_grade", paramsEnabled: false, }, + { + endpoint: + "/v1/extendedattributes/6f7d5be0-f91a-4353-9d31-13983205cbe0/dropdownlistitems", + method: "GET", + dbTable: "aqi_tissue_types", + paramsEnabled: false, + }, + { + endpoint: "/v1/extendedattributes/65d94fac-aac5-498f-bc73-b63a322ce350/dropdownlistitems", + method: "GET", + dbTable: "aqi_sampling_agency", + paramsEnabled: false, + }, { endpoint: "/v1/samplinglocations", method: "GET", @@ -175,6 +191,24 @@ export class CronJobService { */ private getUpdatePayload(dbTable: string, record: any): any { switch (dbTable) { + case "aqi_tissue_types": + return { + aqi_tissue_types_id: record.id, + custom_id: record.customId, + create_user_id: "EnMoDS", + create_utc_timestamp: new Date(), + update_user_id: "EnMoDS", + update_utc_timestamp: new Date(), + }; + case "aqi_sampling_agency": + return { + aqi_sampling_agency_id: record.id, + custom_id: record.customId, + create_user_id: "EnMoDS", + create_utc_timestamp: new Date(), + update_user_id: "EnMoDS", + update_utc_timestamp: new Date(), + }; case "aqi_field_visits": return { aqi_field_visit_start_time: new Date(record.startTime), @@ -237,6 +271,24 @@ export class CronJobService { */ private getCreatePayload(dbTable: string, record: any): any { switch (dbTable) { + case "aqi_tissue_types": + return { + aqi_tissue_types_id: record.id, + custom_id: record.customId, + create_user_id: "EnMoDS", + create_utc_timestamp: new Date(), + update_user_id: "EnMoDS", + update_utc_timestamp: new Date(), + }; + case "aqi_sampling_agency": + return { + aqi_sampling_agency_id: record.id, + custom_id: record.customId, + create_user_id: "EnMoDS", + create_utc_timestamp: new Date(), + update_user_id: "EnMoDS", + update_utc_timestamp: new Date(), + }; case "aqi_field_visits": return { [`${dbTable}_id`]: record.id, @@ -474,6 +526,23 @@ export class CronJobService { modificationTime, }; }; + const filterEELists = (obj: any): any => { + const { id, customId } = obj; + const create_user_id = "EnMoDs"; + const create_utc_timestamp = new Date().toISOString(); + const update_user_id = "EnMoDs"; + const update_utc_timestamp = new Date().toISOString(); + + return { + id, + customId, + create_user_id, + create_utc_timestamp, + update_user_id, + update_utc_timestamp, + }; + }; + const filterArray = (array: any): any => { if (endpoint == "/v1/tags") { return array.map(filterNameAttributes); @@ -485,6 +554,13 @@ export class CronJobService { return array.map(filterSpecimenAttributes); } else if (endpoint == "/v1/analysismethods") { return array.map(filerAnalysisMethodAttributes); + } else if ( + endpoint == + "/v1/extendedattributes/6f7d5be0-f91a-4353-9d31-13983205cbe0/dropdownlistitems" || + endpoint == + "/v1/extendedattributes/65d94fac-aac5-498f-bc73-b63a322ce350/dropdownlistitems" + ) { + return array.map(filterEELists); } else { return array.map(filterAttributes); } @@ -499,10 +575,10 @@ export class CronJobService { grab all the files from the DB and S3 bucket that have a status of QUEUED for each file returned, change the status to INPROGRESS and go to the parser // */ - // if (!this.dataPullDownComplete) { - // this.logger.warn("Data pull down from AQSS did not complete"); - // return; - // } + if (!this.dataPullDownComplete) { + this.logger.warn("Data pull down from AQSS did not complete"); + return; + } let filesToValidate = await this.fileParser.getQueuedFiles(); @@ -517,24 +593,39 @@ export class CronJobService { } async processFiles(files) { - const wait = (ms: number) => - new Promise((resolve) => setTimeout(resolve, ms)); - for (const file of files) { - const fileBinary = await this.objectStore.getFileData(file.file_name); - this.logger.log(`SENT FILE: ${file.file_name}`); - - await this.fileParser.parseFile( - fileBinary, - file.file_name, - file.original_file_name, - file.submission_id, - file.file_operation_code, - ); + if (this.isProcessing){ + this.logger.log("Skipping cron execution: Already processing files."); + return; + } + + this.isProcessing = true; + this.logger.log("Starting to process queued files..."); + + try{ + for (const file of files) { + try { + const fileBinary = await this.objectStore.getFileData(file.file_name); + this.logger.log(`SENT FILE: ${file.file_name}`); + + await this.fileParser.parseFile( + fileBinary, + file.file_name, + file.original_file_name, + file.submission_id, + file.file_operation_code, + ); - this.logger.log(`WAITING FOR PREVIOUS FILE`); - this.logger.log("GOING TO NEXT FILE"); + this.logger.log(`File ${file.file_name} processed successfully.`); + } catch (err) { + this.logger.error(`Error processing file ${file.file_name}: ${err}`); + } + + this.logger.log("GOING TO NEXT FILE"); + } + }finally{ + this.isProcessing = false; + this.dataPullDownComplete = false; + return; } - this.dataPullDownComplete = false; - return; } } diff --git a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts index c70e576..59e84c2 100644 --- a/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts +++ b/backend/src/file_parse_and_validation/file_parse_and_validation.service.ts @@ -37,7 +37,7 @@ const activities: FieldActivities = { LocationID: "", ObservedDateTime: "", ObservedDateTimeEnd: "", - ActivityType: "SAMPLE_ROUTINE", + ActivityType: "", ActivityName: "", ActivityComments: "", SamplingContextTag: "", @@ -709,7 +709,19 @@ export class FileParseValidateService { }); if (customAttributes) { - Object.assign(filteredObj, customAttributes); + if (customAttributes.hasOwnProperty('ActivityType')){ + if (row["DataClassification"] == "VERTICAL_PROFILE"){ + Object.assign(filteredObj, {"ActivityType": "SAMPLE_INTEGRATED_VERTICAL_PROFILE"}) + }else if (row["DataClassification"] == "LAB" || row["DataClassification"] == "FIELD_RESULT"){ + if (row["QCType"] == ""){ + Object.assign(filteredObj, {"ActivityType": "SAMPLE_ROUTINE"}) + }else{ + Object.assign(filteredObj, {"ActivityType": `${row['QCType']}`}) + } + } + }else{ + Object.assign(filteredObj, customAttributes) + } } return filteredObj; @@ -717,23 +729,6 @@ export class FileParseValidateService { } getUniqueWithCounts(data: any[]) { - // const map = new Map< - // string, - // { rec: any; count: number; positions: number[] } - // >(); - - // data.forEach((obj, index) => { - // const key = JSON.stringify(obj); - // if (map.has(key)) { - // const entry = map.get(key)!; - // entry.count++; - // entry.positions.push(index); - // } else { - // map.set(key, { rec: obj, count: 1, positions: [index] }); - // } - // }); - // const dupeCount = Array.from(map.values()); - // return dupeCount; const seen = new Map(); const duplicateDetails = []; @@ -803,7 +798,7 @@ export class FileParseValidateService { "MethodReportingLimit", ]; - const unitFields = ["ResultUnit"]; + const unitFields = "ResultUnit"; // check all datetimes dateTimeFields.forEach((field) => { @@ -812,17 +807,19 @@ export class FileParseValidateService { if (!valid) { let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${field}": "${record[field]} is not valid ISO DateTime"}}`; errorLogs.push(JSON.parse(errorLog)); - } else if (record.hasOwnProperty(field) && !record[field]) { + } + } else if (record.hasOwnProperty(field) && !record[field]) { + if (field == "FieldVisitStartTime" || field == "ObservedDateTime") { let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${field}": "Cannot be empty"}}`; errorLogs.push(JSON.parse(errorLog)); } - } else if (record.hasOwnProperty(field) && !record[field]) { + if ( - field == "FieldVisitStartTime" || - field == "ObservedDateTime" || - field == "AnalyzedDateTime" + (record["DataClassification"] == "LAB_DATA" || + record["DataClassification"] == "SURROGATE_RESULT") && + record["AnalyzedDateTime"] == "" ) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${field}": "Cannot be empty"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${field}": "Cannot be empty for data classification ${record["DataClassification"]}"}}`; errorLogs.push(JSON.parse(errorLog)); } } @@ -842,21 +839,47 @@ export class FileParseValidateService { }); // check all unit fields - unitFields.forEach(async (field) => { - if (record.hasOwnProperty(field) && record[field]) { + if (record.hasOwnProperty(unitFields)) { + if (record[unitFields]){ const present = await this.aqiService.databaseLookup( "aqi_units_xref", - record[field], + record[unitFields], ); + if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${field}": "${record[field]} not found in AQI Units"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${unitFields}": "${record[unitFields]} not found in EnMoDS Units"}}`; errorLogs.push(JSON.parse(errorLog)); } - } else if (record.hasOwnProperty(field) && !record[field]) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${field}": Cannot be empty"}}`; + } + } else if (record.hasOwnProperty(unitFields)) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"${unitFields}": Cannot be empty"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + + if (record.hasOwnProperty("Depth Unit")) { + if (record["Depth Upper"]) { + if (record["Depth Unit"] != "metre") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Depth_Unit": "${record["Depth Unit"]} is not valid unit for Depth. Only 'Metre' is allowed"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + } + } + + if (record.hasOwnProperty("SamplingAgency")) { + if (record["SamplingAgency"] == "") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Sampling Agency": "Cannot be empty"}}`; errorLogs.push(JSON.parse(errorLog)); + } else { + const present = await this.aqiService.databaseLookup( + "aqi_sampling_agency", + record.SamplingAgency, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Sampling Agency": "${record.SamplingAgency} not found in EnMoDS Sampling Agency"}}`; + errorLogs.push(JSON.parse(errorLog)); + } } - }); + } if (record.hasOwnProperty("Project")) { const present = await this.aqiService.databaseLookup( @@ -864,19 +887,24 @@ export class FileParseValidateService { record.Project, ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Project": "${record.Project} not found in AQI Projects"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Project": "${record.Project} not found in EnMoDS Projects"}}`; errorLogs.push(JSON.parse(errorLog)); } } if (record.hasOwnProperty("LocationID")) { - const present = await this.aqiService.databaseLookup( - "aqi_locations", - record.LocationID, - ); - if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Location_ID": "${record.LocationID} not found in AQI Locations"}}`; + if (record["LocationID"] == "") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Location_ID": "Cannot be empty"}}`; errorLogs.push(JSON.parse(errorLog)); + } else { + const present = await this.aqiService.databaseLookup( + "aqi_locations", + record.LocationID, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Location_ID": "${record.LocationID} not found in EnMoDS Locations"}}`; + errorLogs.push(JSON.parse(errorLog)); + } } } @@ -886,7 +914,20 @@ export class FileParseValidateService { record.Preservative, ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Preservative": "${record.Preservative} not found in AQI Preservatives"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Preservative": "${record.Preservative} not found in EnMoDS Preservatives"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + } + + if (record.hasOwnProperty("FieldDeviceType")) { + if ( + (record["DataClassification"] == "FIELD_RESULT" || + record["DataClassification"] == "ACTIVITY_RESULT" || + record["DataClassification"] == "FIELD_SURVEY" || + record["DataClassification"] == "VERTICAL_PROFILE") && + record["FieldDeviceType"] == "" + ) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Field Device Type": "Cannot be empty when data classification is ${record["DataClassification"]}"}}`; errorLogs.push(JSON.parse(errorLog)); } } @@ -897,42 +938,62 @@ export class FileParseValidateService { record.SamplingConextTag, ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Sampling_Context_Tag": "${record.SamplingConextTag} not found in AQI Sampling Context Tags"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Sampling_Context_Tag": "${record.SamplingConextTag} not found in EnMoDS Sampling Context Tags"}}`; errorLogs.push(JSON.parse(errorLog)); } } if (record.hasOwnProperty("CollectionMethod")) { - const present = await this.aqiService.databaseLookup( - "aqi_collection_methods", - record.CollectionMethod, - ); - if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Collection_Method": "${record.CollectionMethod} not found in AQI Collection Methods"}}`; + if ( + (record["DataClassification"] == "LAB" || + record["DataClassification"] == "SURROGATE_RESULT") && + record["CollectionMethod"] == "" + ) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Collection_Method": "Cannot be empty when Data Classification is ${record["DataClassification"]}"}}`; errorLogs.push(JSON.parse(errorLog)); + } else { + if (record["CollectionMethod"] != ""){ + const present = await this.aqiService.databaseLookup( + "aqi_collection_methods", + record.CollectionMethod, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Collection_Method": "${record.CollectionMethod} not found in EnMoDS Collection Methods"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + } } } if (record.hasOwnProperty("Medium")) { - const present = await this.aqiService.databaseLookup( - "aqi_mediums", - record.Medium, - ); - if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Medium": "${record.Medium} not found in AQI Mediums"}}`; + if (record["Medium"] == "") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Medium": "Cannot be empty"}}`; errorLogs.push(JSON.parse(errorLog)); + } else { + const present = await this.aqiService.databaseLookup( + "aqi_mediums", + record.Medium, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Medium": "${record.Medium} not found in EnMoDS Mediums"}}`; + errorLogs.push(JSON.parse(errorLog)); + } } } if (record.hasOwnProperty("ObservedPropertyID")) { - const present = await this.aqiService.databaseLookup( - "aqi_observed_properties", - record.ObservedPropertyID, - ); - if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Observed_Property_ID": "${record.ObservedPropertyID} not found in AQI Observed Properties"}}`; + if (record["ObservedPropertyID"] == "") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Observed_Property_ID": "Cannot be empty"}}`; errorLogs.push(JSON.parse(errorLog)); - errorLog += `ERROR: Row ${index + 2} Observed Property ID ${record.ObservedPropertyID} not found in AQI Observed Properties\n`; + } else { + const present = await this.aqiService.databaseLookup( + "aqi_observed_properties", + record.ObservedPropertyID, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Observed_Property_ID": "${record.ObservedPropertyID} not found in EnMoDS Observed Properties"}}`; + errorLogs.push(JSON.parse(errorLog)); + } } } @@ -945,7 +1006,7 @@ export class FileParseValidateService { record.DetectionCondition.toUpperCase().replace(/ /g, "_"), ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Detection_Condition": "${record.DetectionCondition} not found in AQI Detection Conditions"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Detection_Condition": "${record.DetectionCondition} not found in EnMoDS Detection Conditions"}}`; errorLogs.push(JSON.parse(errorLog)); } } @@ -956,30 +1017,52 @@ export class FileParseValidateService { record.Fraction.toUpperCase(), ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Fraction": "${record.Fraction} not found in AQI Fractions"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Fraction": "${record.Fraction} not found in EnMoDS Fractions"}}`; errorLogs.push(JSON.parse(errorLog)); } } if (record.hasOwnProperty("DataClassification")) { - const present = await this.aqiService.databaseLookup( - "aqi_data_classifications", - record.DataClassification, - ); - if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Data_Classification": "${record.DataClassification} not found in AQI Data Classesifications"}}`; + if (record["DataClassification"] == "") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Data Classification": "Cannot be empty"}}`; + errorLogs.push(JSON.parse(errorLog)); + } else { + const present = await this.aqiService.databaseLookup( + "aqi_data_classifications", + record.DataClassification, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Data_Classification": "${record.DataClassification} not found in EnMoDS Data Classesifications"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + } + + if ( + record["CompositeStat"] != "" && + record["DataClassification"] != "LAB" + ) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Data Classification": "Must be LAB when Composite Stat is porvided."}}`; errorLogs.push(JSON.parse(errorLog)); } } if (record.hasOwnProperty("AnalyzingAgency")) { - const present = await this.aqiService.databaseLookup( - "aqi_laboratories", - record.AnalyzingAgency, - ); - if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Analyzing_Agency": "${record.AnalyzingAgency} not found in AQI Agencies"}}`; + if ( + (record["DataClassification"] == "LAB" || + record["DataClassification"] == "SURROGATE_RESULT") && + record["AnalyzingAgency"] == "" + ) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Analyzing Agency": "Cannot be empty when Data Classification is ${record["DataClassification"]}"}}`; errorLogs.push(JSON.parse(errorLog)); + } else { + const present = await this.aqiService.databaseLookup( + "aqi_laboratories", + record.AnalyzingAgency, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Analyzing_Agency": "${record.AnalyzingAgency} not found in EnMoDS Agencies"}}`; + errorLogs.push(JSON.parse(errorLog)); + } } } @@ -989,7 +1072,7 @@ export class FileParseValidateService { record.ResultStatus, ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Result_Status": "${record.ResultStatus} not found in AQI Result Statuses"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Result_Status": "${record.ResultStatus} not found in EnMoDS Result Statuses"}}`; errorLogs.push(JSON.parse(errorLog)); } } @@ -1000,7 +1083,36 @@ export class FileParseValidateService { record.ResultGrade, ); if (!present) { - let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Result_Grade": "${record.ResultGrade} not found in AQI Result Grades"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Result_Grade": "${record.ResultGrade} not found in EnMoDS Result Grades"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + } + + if (record.hasOwnProperty("TissueType")){ + if (record["Medium"] == "Animal - Fish" && record["TissueType"] == ""){ + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Tissue Type": "Cannot be empty when Medium is Animal - Fish"}}`; + errorLogs.push(JSON.parse(errorLog)); + }else if (record["TissueType"]) { + const present = await this.aqiService.databaseLookup( + "aqi_tissue_types", + record.TissueType, + ); + if (!present) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Tissue Type": "${record.TissueType} not found in EnMoDS Tissue Types"}}`; + errorLogs.push(JSON.parse(errorLog)); + } + } + } + + if (record.hasOwnProperty("SpecimenName")) { + if (record["CompositeStat"] != "" && record["SpecimenName"] == "") { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Specimen Name": "Cannot be empty when Composite Stat is present."}}`; + errorLogs.push(JSON.parse(errorLog)); + } else if ( + record["Medium"] == "Animal - Fish" && + record["SpecimenName"] == "" + ) { + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Specimen Name": "Cannot be empty when Medium is Animal - Fish"}}`; errorLogs.push(JSON.parse(errorLog)); } } @@ -1013,7 +1125,7 @@ export class FileParseValidateService { ]); if (visitExists !== null && visitExists !== undefined) { existingGUIDS["visit"] = visitExists; - let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in AQI Field Visits"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Visit": "Visit for Location ${record.LocationID} at Start Time ${record.FieldVisitStartTime} already exists in EnMoDS Field Visits"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -1024,7 +1136,7 @@ export class FileParseValidateService { ); if (activityExists !== null && activityExists !== undefined) { existingGUIDS["activity"] = activityExists; - let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in AQI Activities"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Activity": "Activity Name ${record.ActivityName} for Field Visit at Start Time ${record.FieldVisitStartTime} already exists in EnMoDS Activities"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -1037,7 +1149,7 @@ export class FileParseValidateService { ]); if (specimenExists !== null && specimenExists !== undefined) { existingGUIDS["specimen"] = specimenExists; - let errorLog = `{"rowNum": ${index + 2}, "type": "WARN", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in AQI Specimen"}}`; + let errorLog = `{"rowNum": ${index + 2}, "type": "ERROR", "message": {"Specimen": "Specimen Name ${record.SpecimenName} for that Acitivity at Start Time ${record.ObservedDateTime} already exists in EnMoDS Specimen"}}`; errorLogs.push(JSON.parse(errorLog)); } @@ -1216,7 +1328,7 @@ export class FileParseValidateService { }; const fieldActivityCustomAttrib: Partial = { - ActivityType: "SAMPLE_ROUTINE", + ActivityType: "", }; /* @@ -1233,13 +1345,14 @@ export class FileParseValidateService { Object.keys(activities), fieldActivityCustomAttrib, ); + let allSpecimens = this.filterFile( allRecords, Object.keys(specimens), null, ); - const allObservations = this.filterFile( + let allObservations = this.filterFile( allRecords, Object.keys(observations), null, @@ -1254,6 +1367,7 @@ export class FileParseValidateService { const uniqueMinistryContacts = Array.from( new Set(allRecords.map((rec) => rec.MinistryContact)), ); + /* * Do the local validation for each section here - if passed then go to the API calls - else create the message/file/email for the errors */ @@ -1270,7 +1384,14 @@ export class FileParseValidateService { file_operation_code, ); - if (localValidationResults[0].some((item) => item.type === "ERROR")) { + const hasError = localValidationResults[0].some( + (item) => item.type === "ERROR", + ); + const hasWarn = localValidationResults[0].some( + (item) => item.type === "WARN", + ); + + if (hasError) { /* * If there are any errors then * Set the file status to 'REJECTED' @@ -1291,9 +1412,9 @@ export class FileParseValidateService { * If there are no errors then * Check to see if there are any WARNINGS * If WARNINGS - * Proceed with the PATCH logic + * Proceed with the PUT logic */ - if (localValidationResults[0].some((item) => item.type === "WARN")) { + if (hasWarn) { let visitInfo = [], expandedVisitInfo = []; let activityInfo = [], @@ -1413,6 +1534,7 @@ export class FileParseValidateService { const uniqueSpecimensWithIDsAndCounts = this.getUniqueWithCounts( allSpecimensWithGUIDS, ); + specimenInfo = await this.specimensJson( uniqueSpecimensWithIDsAndCounts, "put", @@ -1423,8 +1545,9 @@ export class FileParseValidateService { const obj1 = expandedActivityInfo[index]; return { ...obj2, ...obj1 }; }); - const uniqueSpecimensWithCounts = - this.getUniqueWithCounts(allSpecimens); + const uniqueSpecimensWithCounts = this.getUniqueWithCounts( + allSpecimens, + ).filter((item) => item.rec.SpecimenName !== ""); specimenInfo = await this.specimensJson( uniqueSpecimensWithCounts, "post", @@ -1479,7 +1602,7 @@ export class FileParseValidateService { active_ind: false, }, }); - } else { + } else if (!hasError && !hasWarn) { // If there are no errors or warnings await this.fileSubmissionsService.updateFileStatus( file_submission_id, @@ -1546,8 +1669,9 @@ export class FileParseValidateService { const obj1 = expandedActivityInfo[index]; return { ...obj2, ...obj1 }; }); - const uniqueSpecimensWithCounts = - this.getUniqueWithCounts(allSpecimens).filter(item => item.rec.SpecimenName !== ""); + const uniqueSpecimensWithCounts = this.getUniqueWithCounts( + allSpecimens, + ).filter((item) => item.rec.SpecimenName !== ""); let specimenInfo = await this.specimensJson( uniqueSpecimensWithCounts, diff --git a/backend/src/types/types.ts b/backend/src/types/types.ts index 80beacc..1343e02 100644 --- a/backend/src/types/types.ts +++ b/backend/src/types/types.ts @@ -65,7 +65,7 @@ export type FieldActivities = { LocationID: string; ObservedDateTime: string; ObservedDateTimeEnd: string; - ActivityType: "SAMPLE_ROUTINE"; + ActivityType: string; ActivityName: string; ActivityComments: string; SamplingContextTag: string; diff --git a/frontend/src/pages/FileUpload.tsx b/frontend/src/pages/FileUpload.tsx index b8517c8..e012684 100644 --- a/frontend/src/pages/FileUpload.tsx +++ b/frontend/src/pages/FileUpload.tsx @@ -318,23 +318,28 @@ function FileUpload() { ) : ( - + "" )} - + {fileStatusCodes.items[index] == "ACCEPTED" || + fileStatusCodes.items[index] == "REJECTED" || + fileStatusCodes.items[index] == "INPROGRESS" || + fileStatusCodes.items[index] == "QUEUED" ? ( + "" + ) : ( + + )} diff --git a/migrations/sql/V1.0.3__create_code_tables.sql b/migrations/sql/V1.0.3__create_code_tables.sql index 1026d25..71f6597 100644 --- a/migrations/sql/V1.0.3__create_code_tables.sql +++ b/migrations/sql/V1.0.3__create_code_tables.sql @@ -107,6 +107,22 @@ CREATE TABLE IF NOT EXISTS enmods.aqi_result_grade ( update_user_id varchar(200) NOT NULL, update_utc_timestamp timestamp NOT NULL ); +CREATE TABLE IF NOT EXISTS enmods.aqi_tissue_types( + aqi_tissue_types_id UUID PRIMARY KEY NOT NULL, + custom_id varchar(200) NOT NULL, + create_user_id varchar(200) NOT NULL, + create_utc_timestamp timestamp NOT NULL, + update_user_id varchar(200) NOT NULL, + update_utc_timestamp timestamp NOT NULL +); +CREATE TABLE IF NOT EXISTS enmods.aqi_sampling_agency( + aqi_sampling_agency_id UUID PRIMARY KEY NOT NULL, + custom_id varchar(200) NOT NULL, + create_user_id varchar(200) NOT NULL, + create_utc_timestamp timestamp NOT NULL, + update_user_id varchar(200) NOT NULL, + update_utc_timestamp timestamp NOT NULL +); CREATE TABLE IF NOT EXISTS enmods.aqi_field_visits( aqi_field_visits_id UUID PRIMARY KEY NOT NULL, aqi_field_visit_start_time timestamptz NOT NULL,