#3745 - Modify process that reads SIN & CRA verification response fil…

…es (#3835) ***Acceptance Criteria*** - [x] Jobs must continue processing remaining records in a file after it fails to process a record. - [x] SIN and CRA response files will need to be archived after processing even where there are failed records. (IMB staff will need to monitor the jobs to confirm if errors occur processing SIMS response files and address them accordingly) - [x] Disable error reporting when SIMS Reference_IDX fails to match. - [x] If records are identified as SIMS records (based on the project area information being present), errors will be generated as usual. - [x] If records are identified as SFAS records (based on the project area information NOT being present), records should be skipped. - [x] For the SIN validation ([source reference](https://github.com/bcgov/SIMS/blob/8864eaa8a5f62360f5e3c91c6de06f319e008250/sources/packages/backend/libs/integrations/src/services/sin-validation/sin-validation.service.ts#L119)), if the `referenceIndex` is not found on SIMS, assume the record is from SFAS and do not log as an error. - [x] For income verifications, the records should already be skipped. Execute a manual test to ensure that a file with records that do not contain the `VERIFICATION_ID` in the free project area is skipped. Consider creating an E2E to execute the test. - [x] Demo for both cases with a sample file.
bcgov · Oct 30, 2024 · 20fcda4 · 20fcda4
1 parent 5bb5050
commit 20fcda4
Show file tree

Hide file tree

Showing 7 changed files with 270 additions and 5 deletions.
diff --git a/...rc/processors/schedulers/cra-integration/_tests_/cra-receive-files/CRA_200_PBCSA00000.TXT b/...rc/processors/schedulers/cra-integration/_tests_/cra-receive-files/CRA_200_PBCSA00000.TXT
@@ -0,0 +1,10 @@
+7200                        20240101 BCSAP00000      00000001                                                                                     0
+720110000000120240001NAME                           JOHN                         1999010110000000000                                              0
+7201100000001202400025077 111 AA                  DUMMY                       OO V0Y 0V0                                                          0
+7201100000001202400220001010101010101010010000VERIFICATION_ID:CRA_INCOME_VERIFICATIONBCSA                                                                     0
+720110000000120240150000050000                                                                                                              15000 0
+7201100000001202400220001010101010101010010000010101010100100010101010100100010101010BCSA                                                                     0
+720110000000120241010000050000                                                                                                              10100 0
+72011000000012024A010BC                                                                                                                           0
+72011000000012024A015BC                                                                                                                           0
+72011000000012024A030N00000000                                                                                                                    0
diff --git a/...cessors/schedulers/cra-integration/_tests_/cra-response-integration.scheduler.e2e-spec.ts b/...cessors/schedulers/cra-integration/_tests_/cra-response-integration.scheduler.e2e-spec.ts
@@ -0,0 +1,98 @@
+import { createMock, DeepMocked } from "@golevelup/ts-jest";
+import { INestApplication } from "@nestjs/common";
+import { QueueNames } from "@sims/utilities";
+import {
+  createTestingAppModule,
+  describeProcessorRootTest,
+} from "../../../../../test/helpers";
+import {
+  E2EDataSources,
+  createE2EDataSources,
+  createFakeCRAIncomeVerification,
+  saveFakeApplication,
+  saveFakeStudent,
+} from "@sims/test-utils";
+import * as Client from "ssh2-sftp-client";
+import * as path from "path";
+import { CRAResponseIntegrationScheduler } from "../cra-response-integration.scheduler";
+import {
+  createFileFromStructuredRecords,
+  getStructuredRecords,
+  mockDownloadFiles,
+} from "@sims/test-utils/mocks";
+import { Job } from "bull";
+import { ApplicationStatus } from "@sims/sims-db";
+
+const CRA_FILENAME = "CRA_200_PBCSA00000.TXT";
+
+describe(describeProcessorRootTest(QueueNames.CRAResponseIntegration), () => {
+  let app: INestApplication;
+  let processor: CRAResponseIntegrationScheduler;
+  let db: E2EDataSources;
+  let sftpClientMock: DeepMocked<Client>;
+  let craResponseFolder: string;
+
+  beforeAll(async () => {
+    craResponseFolder = path.join(__dirname, "cra-receive-files");
+    process.env.CRA_RESPONSE_FOLDER = craResponseFolder;
+    const { nestApplication, dataSource, sshClientMock } =
+      await createTestingAppModule();
+    app = nestApplication;
+    db = createE2EDataSources(dataSource);
+    sftpClientMock = sshClientMock;
+    processor = app.get(CRAResponseIntegrationScheduler);
+  });
+
+  beforeEach(async () => {
+    jest.clearAllMocks();
+  });
+
+  it("Should process SIN response file ignoring non-SIMS records when the file contains responses from requests that were not created by SIMS.", async () => {
+    // Arrange.
+    const student = await saveFakeStudent(db.dataSource);
+
+    const application = await saveFakeApplication(
+      db.dataSource,
+      { student },
+      { applicationStatus: ApplicationStatus.InProgress },
+    );
+
+    // Create CRA income verifications for student.
+    const studentCRAIncomeVerification = createFakeCRAIncomeVerification({
+      application,
+    });
+    await db.craIncomeVerification.save([studentCRAIncomeVerification]);
+    // Queued job.
+    const job = createMock<Job<void>>();
+    mockDownloadFiles(sftpClientMock, [CRA_FILENAME]);
+
+    mockDownloadFiles(sftpClientMock, [CRA_FILENAME], (fileContent: string) => {
+      const file = getStructuredRecords(fileContent);
+      file.records[2] = file.records[2].replace(
+        "CRA_INCOME_VERIFICATION",
+        studentCRAIncomeVerification.id.toString().padStart(9, "0"),
+      );
+      return createFileFromStructuredRecords(file);
+    });
+
+    // Act
+    const processResult = await processor.processResponses(job);
+    // Assert
+    const downloadedFile = path.join(
+      process.env.CRA_RESPONSE_FOLDER,
+      CRA_FILENAME,
+    );
+
+    // Assert
+    expect(processResult).toStrictEqual([
+      {
+        processSummary: [
+          `Processing file ${downloadedFile}.`,
+          "File contains 2 verifications.",
+          "Processed income verification. Total income record line 5. Status record from line 4.",
+        ],
+        errorsSummary: [],
+      },
+    ]);
+  });
+});
diff --git a/...sdc-integration/sin-validation-integration/_tests_/sin-receive-files/PCSLP.PBC.BC0000.ISR b/...sdc-integration/sin-validation-integration/_tests_/sin-receive-files/PCSLP.PBC.BC0000.ISR
@@ -0,0 +1,4 @@
+00100500220220921BC1       
+0026000000014Y100000001YYNY
+0026000000025Y100000002YYNY
+999000014000000200000003
diff --git a/...ion-integration/_tests_/sin-validation-process-response-integration.scheduler.e2e-spec.ts b/...ion-integration/_tests_/sin-validation-process-response-integration.scheduler.e2e-spec.ts
@@ -0,0 +1,150 @@
+import { createMock, DeepMocked } from "@golevelup/ts-jest";
+import { INestApplication } from "@nestjs/common";
+import { QueueNames } from "@sims/utilities";
+import {
+  createTestingAppModule,
+  describeProcessorRootTest,
+} from "../../../../../../test/helpers";
+import {
+  E2EDataSources,
+  createE2EDataSources,
+  saveFakeStudent,
+} from "@sims/test-utils";
+import * as Client from "ssh2-sftp-client";
+import * as path from "path";
+import { SINValidationResponseIntegrationScheduler } from "../sin-validation-process-response-integration.scheduler";
+import {
+  createFileFromStructuredRecords,
+  getStructuredRecords,
+  mockDownloadFiles,
+} from "@sims/test-utils/mocks";
+import { Job } from "bull";
+
+const SIN_VALIDATION_FILENAME = "PCSLP.PBC.BC0000.ISR";
+
+describe(
+  describeProcessorRootTest(QueueNames.SINValidationResponseIntegration),
+  () => {
+    let app: INestApplication;
+    let processor: SINValidationResponseIntegrationScheduler;
+    let db: E2EDataSources;
+    let sftpClientMock: DeepMocked<Client>;
+    let sinValidationResponseFolder: string;
+
+    beforeAll(async () => {
+      sinValidationResponseFolder = path.join(__dirname, "sin-receive-files");
+      process.env.ESDC_RESPONSE_FOLDER = sinValidationResponseFolder;
+      const { nestApplication, dataSource, sshClientMock } =
+        await createTestingAppModule();
+      app = nestApplication;
+      db = createE2EDataSources(dataSource);
+      sftpClientMock = sshClientMock;
+      processor = app.get(SINValidationResponseIntegrationScheduler);
+    });
+
+    beforeEach(async () => {
+      jest.clearAllMocks();
+    });
+
+    it("Should skip process SIN response file when no SIN validation was updated because the record id is already present.", async () => {
+      // Arrange
+      // Create a SIN record with REFERENCE_IDX = 600000001
+      const validSinStudent = await saveFakeStudent(db.dataSource, undefined, {
+        sinValidationInitialValue: {
+          sin: "100000001",
+          isValidSIN: true,
+        },
+      });
+
+      // Queued job.
+      const job = createMock<Job<void>>();
+      mockDownloadFiles(sftpClientMock, [SIN_VALIDATION_FILENAME]);
+
+      mockDownloadFiles(
+        sftpClientMock,
+        [SIN_VALIDATION_FILENAME],
+        (fileContent: string) => {
+          const file = getStructuredRecords(fileContent);
+          file.records[0] = file.records[0].replace(
+            // Assuming the first 3 characters are some identifier, replace everything between that and position 12
+            file.records[0].substring(3, 12),
+            validSinStudent.sinValidation.id.toString().padStart(9, "0"),
+          );
+          return createFileFromStructuredRecords(file);
+        },
+      );
+
+      // Act
+      const processResult = await processor.processSINValidationResponse(job);
+      // Assert
+      const downloadedFile = path.join(
+        process.env.ESDC_RESPONSE_FOLDER,
+        SIN_VALIDATION_FILENAME,
+      );
+
+      // Assert
+      expect(processResult).toStrictEqual([
+        {
+          processSummary: [
+            `Processing file ${downloadedFile}.`,
+            "File contains 2 SIN validations.",
+            "Processed SIN validation record from line 2: No SIN validation was updated because the record id is already present and this is not the most updated.",
+            "Processed SIN validation record from line 3: Not able to find the SIN validation on line number 3 to be updated with the ESDC response.",
+          ],
+          errorsSummary: [],
+        },
+      ]);
+    });
+
+    it("Should update one SIN validation record and skip one when one SIN response is from SIMS and the other is from SFAS.", async () => {
+      // Arrange
+      // Create a SIN record with REFERENCE_IDX = 600000002 and dateReceived = null to process the SIN validation record updated.
+      const validSinStudent = await saveFakeStudent(db.dataSource, undefined, {
+        sinValidationInitialValue: {
+          sin: "100000002",
+          isValidSIN: false,
+          dateReceived: null,
+        },
+      });
+
+      // Queued job.
+      const job = createMock<Job<void>>();
+      mockDownloadFiles(sftpClientMock, [SIN_VALIDATION_FILENAME]);
+
+      mockDownloadFiles(
+        sftpClientMock,
+        [SIN_VALIDATION_FILENAME],
+        (fileContent: string) => {
+          const file = getStructuredRecords(fileContent);
+          file.records[0] = file.records[0].replace(
+            // Assuming the first 3 characters are some identifier, replace everything between that and position 12
+            file.records[0].substring(3, 12),
+            validSinStudent.sinValidation.id.toString().padStart(9, "0"),
+          );
+          return createFileFromStructuredRecords(file);
+        },
+      );
+
+      // Act
+      const processResult = await processor.processSINValidationResponse(job);
+      // Assert
+      const downloadedFile = path.join(
+        process.env.ESDC_RESPONSE_FOLDER,
+        SIN_VALIDATION_FILENAME,
+      );
+
+      // Assert
+      expect(processResult).toStrictEqual([
+        {
+          processSummary: [
+            `Processing file ${downloadedFile}.`,
+            "File contains 2 SIN validations.",
+            "Processed SIN validation record from line 2: SIN validation record updated.",
+            "Processed SIN validation record from line 3: Not able to find the SIN validation on line number 3 to be updated with the ESDC response.",
+          ],
+          errorsSummary: [],
+        },
+      ]);
+    });
+  },
+);
diff --git a/...end/libs/integrations/src/esdc-integration/sin-validation/models/sin-validation-models.ts b/...end/libs/integrations/src/esdc-integration/sin-validation/models/sin-validation-models.ts
@@ -57,6 +57,6 @@ export interface SINValidationRecord {
  * description of the operation executed.
  */
 export interface SINValidationUpdateResult {
-  record: SINValidation;
+  record?: SINValidation;
   operationDescription: string;
 }
diff --git a/.../packages/backend/libs/integrations/src/services/sin-validation/sin-validation.service.ts b/.../packages/backend/libs/integrations/src/services/sin-validation/sin-validation.service.ts
@@ -121,9 +121,9 @@ export class SINValidationService extends RecordDataModelService<SINValidation>
           .getOne();
 
         if (!existingValidation) {
-          throw new Error(
-            `Not able to find the SIN validation id ${validationResponse.referenceIndex} to be updated with the ESDC response.`,
-          );
+          return {
+            operationDescription: `Not able to find the SIN validation on line number ${validationResponse.lineNumber} to be updated with the ESDC response.`,
+          };
         }
 
         const sinValidationNeverUpdated = !existingValidation.dateReceived;

diff --git a/sources/packages/backend/libs/test-utils/src/factories/sin-validation.ts b/sources/packages/backend/libs/test-utils/src/factories/sin-validation.ts
@@ -23,7 +23,10 @@ export function createFakeSINValidation(
     options?.initialValue?.sin ??
     faker.datatype.number({ min: 100000000, max: 899999999 }).toString();
   sinValidation.dateSent = now;
-  sinValidation.dateReceived = now;
+  sinValidation.dateReceived =
+    options?.initialValue?.dateReceived !== undefined
+      ? options.initialValue.dateReceived
+      : now;
   sinValidation.fileSent = null;
   sinValidation.fileReceived = null;
   sinValidation.givenNameSent = null;