Merge pull request #34 from bcgov/EMSEDT-185_Search_Functionality

EMSEDT-185: Search Functionality

backend/prisma/schema.prisma

@@ -24,6 +24,7 @@ model submission_status_code {
 model file_submission {
   submission_id          String                 @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
   file_name              String                 @db.VarChar(200)
+  original_file_name     String                 @db.VarChar(200)
   submission_date        DateTime               @db.Timestamp(6)
   submitter_user_id      String                 @db.VarChar(200)
   submission_status_code String                 @db.VarChar(10)
@@ -242,11 +243,15 @@ model file_operation_codes {
 }
 
 model file_error_logs {
-  file_error_log_id  String           @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
-  file_submission_id String?          @db.Uuid
-  file_name          String?          @db.VarChar(200)
-  error_log          Json?
-  file_submission    file_submission? @relation(fields: [file_submission_id], references: [submission_id], onDelete: NoAction, onUpdate: NoAction, map: "file_submission_id_fk")
+  file_error_log_id    String           @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid
+  file_submission_id   String?          @db.Uuid
+  file_name            String?          @db.VarChar(200)
+  original_file_name   String?          @db.VarChar(200)
+  file_operation_code  String?          @db.VarChar(200)
+  ministry_contact     String?          @db.VarChar(200)
+  error_log            Json?
+  create_utc_timestamp DateTime         @db.Timestamp(6)
+  file_submission      file_submission? @relation(fields: [file_submission_id], references: [submission_id], onDelete: NoAction, onUpdate: NoAction, map: "file_submission_id_fk")
 }
 
 model aqi_data_classifications {

backend/src/app.module.ts

@@ -25,6 +25,7 @@ import { FileParseValidateModule } from "./file_parse_and_validation/file_parse_
 import { FtpModule } from "./ftp/ftp.module";
 import { FileValidationModule } from './file_validation/file_validation.module';
 import { ObjectStoreModule } from "./objectStore/objectStore.module";
+import { FileErrorLogsModule } from './file_error_logs/file_error_logs.module';
 
 const DB_HOST = process.env.POSTGRES_HOST || "localhost";
 const DB_USER = process.env.POSTGRES_USER || "postgres";
@@ -72,7 +73,8 @@ function getMiddlewares() {
     AqiApiModule,
     FtpModule,
     FileValidationModule,
-    ObjectStoreModule
+    ObjectStoreModule,
+    FileErrorLogsModule
   ],
   controllers: [AppController, MetricsController, HealthController],
   providers: [AppService, CronJobService],

backend/src/aqi_api/aqi_api.service.ts

@@ -124,8 +124,7 @@ export class AqiApiService {
         },
       });
 
-      await this.wait(9);
-      console.log(response.data.id)
+      await this.wait(10);
 
       const obsResultResponse = await axios.get(
         `${process.env.AQI_BASE_URL}/v2/observationimports/${response.data.id}/result`,

backend/src/cron-job/cron-job.service.ts

@@ -4,7 +4,6 @@ import { error } from "winston";
 import { Cron, CronExpression } from "@nestjs/schedule";
 import { PrismaService } from "nestjs-prisma";
 import { FileParseValidateService } from "src/file_parse_and_validation/file_parse_and_validation.service";
-import { FileSubmissionsService } from "src/file_submissions/file_submissions.service";
 import { ObjectStoreService } from "src/objectStore/objectStore.service";
 
 /**
@@ -21,7 +20,6 @@ export class CronJobService {
   constructor(
     private prisma: PrismaService,
     private readonly fileParser: FileParseValidateService,
-    private readonly fileSubmissionsService: FileSubmissionsService,
     private readonly objectStore: ObjectStoreService,
   ) {
     this.tableModels = new Map<string, any>([
@@ -433,10 +431,10 @@ export class CronJobService {
       grab all the files from the DB and S3 bucket that have a status of QUEUED
       for each file returned, change the status to INPROGRESS and go to the parser
     */
-    // if (!this.dataPullDownComplete) {
-    //   this.logger.warn("Data pull down from AQSS did not complete");
-    //   return;
-    // }
+    if (!this.dataPullDownComplete) {
+      this.logger.warn("Data pull down from AQSS did not complete");
+      return;
+    }
 
     let filesToValidate = await this.fileParser.getQueuedFiles();
 
@@ -450,6 +448,7 @@ export class CronJobService {
         this.fileParser.parseFile(
           fileBinary,
           file.file_name,
+          file.original_file_name,
           file.submission_id,
           file.file_operation_code,
         );

backend/src/file_error_logs/dto/create-file_error_log.dto.ts

@@ -0,0 +1,13 @@
+import { PickType } from "@nestjs/swagger";
+import { FileErrorLogDto } from "./file_error_logs.dto";
+
+export class CreateFileErrorLogDto extends PickType(FileErrorLogDto, [
+    'file_error_log_id',
+    'file_submission_id',
+    'file_name',
+    'original_file_name',
+    'file_operation_code',
+    'ministry_contact',
+    'error_log',
+    'create_utc_timestamp'
+] as const) {}

backend/src/file_error_logs/dto/file_error_logs.dto.ts

@@ -0,0 +1,43 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class FileErrorLogDto {
+  @ApiProperty({
+    description: "File error log ID",
+  })
+  file_error_log_id: string;
+
+  @ApiProperty({
+    description: "File submission ID",
+  })
+  file_submission_id: string;
+
+  @ApiProperty({
+    description: "File name",
+  })
+  file_name: string;
+
+  @ApiProperty({
+    description: "original file name",
+  })
+  original_file_name: string;
+
+  @ApiProperty({
+    description: "Error log data",
+  })
+  error_log: string;
+
+  @ApiProperty({
+    description: 'The operation the file was submitted for',
+  })
+  file_operation_code: string;
+
+  @ApiProperty({
+    description: 'The ministry contact that needs to be notified',
+  })
+  ministry_contact: string;
+
+  @ApiProperty({
+    description: 'When the user created the record',
+  })
+  create_utc_timestamp: Date;
+}

backend/src/file_error_logs/dto/update-file_error_log.dto.ts

@@ -0,0 +1,3 @@
+import { CreateFileErrorLogDto } from './create-file_error_log.dto';
+
+export class UpdateFileErrorLogDto extends (CreateFileErrorLogDto) {}

backend/src/file_error_logs/entities/file_error_log.entity.ts

@@ -0,0 +1 @@
+export class FileErrorLog {}

backend/src/file_error_logs/file_error_logs.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FileErrorLogsController } from './file_error_logs.controller';
+import { FileErrorLogsService } from './file_error_logs.service';
+
+describe('FileErrorLogsController', () => {
+  let controller: FileErrorLogsController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [FileErrorLogsController],
+      providers: [FileErrorLogsService],
+    }).compile();
+
+    controller = module.get<FileErrorLogsController>(FileErrorLogsController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

backend/src/file_error_logs/file_error_logs.controller.ts

@@ -0,0 +1,55 @@
+import {
+  Controller,
+  Get,
+  Post,
+  Body,
+  Patch,
+  Param,
+  Delete,
+  UseGuards,
+} from "@nestjs/common";
+import { FileErrorLogsService } from "./file_error_logs.service";
+import { CreateFileErrorLogDto } from "./dto/create-file_error_log.dto";
+import { UpdateFileErrorLogDto } from "./dto/update-file_error_log.dto";
+import { ApiTags } from "@nestjs/swagger";
+import { JwtRoleGuard } from "src/auth/jwtrole.guard";
+import { JwtAuthGuard } from "src/auth/jwtauth.guard";
+import { Roles } from "src/auth/decorators/roles.decorators";
+import { Role } from "src/enum/role.enum";
+
+@ApiTags("file_error_logs")
+@Controller({ path: "file_error_logs", version: "1" })
+@UseGuards(JwtAuthGuard)
+@UseGuards(JwtRoleGuard)
+@Roles(Role.ENMODS_ADMIN)
+export class FileErrorLogsController {
+  constructor(private readonly fileErrorLogsService: FileErrorLogsService) {}
+
+  @Post()
+  create(@Body() createFileErrorLogDto: CreateFileErrorLogDto) {
+    return this.fileErrorLogsService.create(createFileErrorLogDto);
+  }
+
+  @Get()
+  findAll() {
+    return this.fileErrorLogsService.findAll();
+  }
+
+  @Get(":file_submission_id")
+  findOne(@Param("file_submission_id") id: string): Promise<string> {
+    return this.fileErrorLogsService.findOne(id);
+  }
+
+  @Patch(":id")
+  update(
+    @Param("id") id: string,
+    @Body() updateFileErrorLogDto: UpdateFileErrorLogDto,
+  ) {
+    return this.fileErrorLogsService.update(+id, updateFileErrorLogDto);
+  }
+
+  @Delete(":id")
+  remove(@Param("id") id: string) {
+    return this.fileErrorLogsService.remove(+id);
+  }
+}

backend/src/file_error_logs/file_error_logs.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { FileErrorLogsService } from './file_error_logs.service';
+import { FileErrorLogsController } from './file_error_logs.controller';
+
+@Module({
+  controllers: [FileErrorLogsController],
+  providers: [FileErrorLogsService],
+})
+export class FileErrorLogsModule {}

backend/src/file_error_logs/file_error_logs.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { FileErrorLogsService } from './file_error_logs.service';
+
+describe('FileErrorLogsService', () => {
+  let service: FileErrorLogsService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [FileErrorLogsService],
+    }).compile();
+
+    service = module.get<FileErrorLogsService>(FileErrorLogsService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

backend/src/file_error_logs/file_error_logs.service.ts

@@ -0,0 +1,77 @@
+import { Injectable } from "@nestjs/common";
+import { CreateFileErrorLogDto } from "./dto/create-file_error_log.dto";
+import { UpdateFileErrorLogDto } from "./dto/update-file_error_log.dto";
+import { PrismaService } from "nestjs-prisma";
+
+@Injectable()
+export class FileErrorLogsService {
+  constructor(private prisma: PrismaService) {}
+
+  create(createFileErrorLogDto: CreateFileErrorLogDto) {
+    return "This action adds a new fileErrorLog";
+  }
+
+  findAll() {
+    return `This action returns all fileErrorLogs`;
+  }
+
+  async findOne(file_submission_id: string): Promise<string> {
+    const fileLogs = await this.prisma.file_error_logs.findMany({
+      where: {
+        file_submission_id: file_submission_id,
+      },
+    });
+
+    const formattedMessage = formulateErrorFile(fileLogs);
+    return formattedMessage;
+  }
+
+  update(id: number, updateFileErrorLogDto: UpdateFileErrorLogDto) {
+    return `This action updates a #${id} fileErrorLog`;
+  }
+
+  remove(id: number) {
+    return `This action removes a #${id} fileErrorLog`;
+  }
+}
+
+function formulateErrorFile(logs: any) {
+  let formattedMessages = "";
+  const [date, timeWithZ] = new Date(logs[0].create_utc_timestamp)
+    .toISOString()
+    .split("T");
+  const time = timeWithZ.replace("Z", "");
+  let fileOperation = ""
+
+  if (logs[0].file_operation_code === 'VALIDATE'){
+    fileOperation = "True";
+  }else{
+    fileOperation = "False";
+  }
+
+  formattedMessages =
+    `User's Original File: ${logs[0].original_file_name}\n` +
+    `${date} ${time}\n\n` +
+    `QA Only: ${fileOperation}\n\n` +
+    `The following warnings/errors were found during the validation/import of the data.\n` +
+    `The data will need to be corrected and uploaded again for validation/import to ENMODS.\n` +
+    `If you have any questions, please contact the ministry contact(s) listed below.\n\n` +
+    `-----------------------------------------------------------------------\n` +
+    `Ministry Contact: ${logs[0].ministry_contact}\n` +
+    `-----------------------------------------------------------------------\n\n`;
+
+  logs[0].error_log.forEach((log) => {
+    const rowNum = log.rowNum;
+
+    for (const [key, msg] of Object.entries(log.message)) {
+      formattedMessages += `${log.type}: Row ${rowNum}: ${key} - ${msg}\n`;
+    }
+  });
+
+  if (logs[0].error_log.length >= 1) {
+    formattedMessages +=
+      "\nData was not updated in ENMODS due to errors found in the submission file. Please correct the data and resubmit.";
+  }
+
+  return formattedMessages;
+}

backend/src/file_parse_and_validation/file_parse_and_validation.module.ts

@@ -3,10 +3,11 @@ import { HttpModule } from '@nestjs/axios';
 import { FileParseValidateService } from './file_parse_and_validation.service';
 import { AqiApiService } from 'src/aqi_api/aqi_api.service';
 import { FileSubmissionsService } from 'src/file_submissions/file_submissions.service';
+import { ObjectStoreModule } from 'src/objectStore/objectStore.module';
 
 @Module({
   providers: [FileParseValidateService, FileSubmissionsService, AqiApiService],
   exports: [FileParseValidateService],
-  imports: [HttpModule]
+  imports: [HttpModule, ObjectStoreModule]
 })
 export class FileParseValidateModule {}

backend/src/file_parse_and_validation/file_parse_and_validation.service.ts

@@ -876,6 +876,7 @@ export class FileParseValidateService {
   async parseFile(
     file: string,
     fileName: string,
+    originalFileName: string,
     file_submission_id: string,
     file_operation_code: string,
   ) {
@@ -943,6 +944,7 @@ export class FileParseValidateService {
         fileName,
       );
 
+      const uniqueMinistryContacts = Array.from(new Set(allRecords.map(rec => rec.MinistryContact)))
       /*
        * Do the local validation for each section here - if passed then go to the API calls - else create the message/file/email for the errors
        */
@@ -971,13 +973,16 @@ export class FileParseValidateService {
         const file_error_log_data = {
           file_submission_id: file_submission_id,
           file_name: fileName,
+          original_file_name: originalFileName,
+          file_operation_code: file_operation_code,
+          ministry_contact: uniqueMinistryContacts.join(', '),
           error_log: localValidationResults,
+          create_utc_timestamp: new Date(),
         };
 
         await this.prisma.file_error_logs.create({
           data: file_error_log_data,
         });
-
         return;
       } else if (!(await localValidationResults).includes("ERROR")) {
         await this.fileSubmissionsService.updateFileStatus(