#3869 - Unzip federal restrictions file

sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.integration.service.ts

@@ -20,7 +20,9 @@ export class FedRestrictionIntegrationService extends SFTPIntegrationBase<
   async downloadResponseFile(
     remoteFilePath: string,
   ): Promise<FedRestrictionFileRecord[]> {
-    const fileLines = await this.downloadResponseFileLines(remoteFilePath);
+    const fileLines = await this.downloadResponseFileLines(remoteFilePath, {
+      checkIfZipFile: true,
+    });
     return fileLines.map((line, index) => {
       return new FedRestrictionFileRecord(line, index + 1);
     });

sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts

@@ -54,7 +54,7 @@ export class FedRestrictionProcessingService {
     const auditUser = this.systemUsersService.systemUser;
     // Get the list of all files from SFTP ordered by file name.
     const fileSearch = new RegExp(
-      `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*$`,
+      `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*.zip$`,
       "i",
     );
 

sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts

@@ -8,12 +8,12 @@ import {
   END_OF_LINE,
   getFileNameAsExtendedCurrentTimestamp,
   convertToASCII,
-  FILE_DEFAULT_ENCODING,
 } from "@sims/utilities";
 import {
   LINE_BREAK_SPLIT_REGEX,
   SFTP_ARCHIVE_DIRECTORY,
 } from "@sims/integrations/constants";
+import * as AdmZip from "adm-zip";
 
 /**
  * Provides the basic features to enable the SFTP integration.
@@ -132,6 +132,19 @@ export abstract class SFTPIntegrationBase<DownloadType> {
     options: { checkIfFileExist: boolean },
   ): Promise<string[] | false>;
 
+  /**
+   * Downloads the file specified on 'fileName' parameter from the
+   * SFAS integration folder on the SFTP.
+   * @param remoteFilePath full remote file path with file name.
+   * @param options download file options.
+   * - `checkIfZipFile` when set to true, the expectation is to receive a zip file.
+   * @returns parsed records from the file.
+   */
+  protected async downloadResponseFileLines(
+    remoteFilePath: string,
+    options: { checkIfZipFile: boolean },
+  ): Promise<string[]>;
+
   /**
    * Downloads the file specified on 'fileName' parameter from the
    * SFAS integration folder on the SFTP.
@@ -142,7 +155,7 @@ export abstract class SFTPIntegrationBase<DownloadType> {
    */
   protected async downloadResponseFileLines(
     remoteFilePath: string,
-    options?: { checkIfFileExist: boolean },
+    options?: { checkIfFileExist: boolean; checkIfZipFile: boolean },
   ): Promise<string[] | false> {
     this.logger.log(`Downloading file ${remoteFilePath}.`);
     let client: Client;
@@ -154,10 +167,20 @@ export abstract class SFTPIntegrationBase<DownloadType> {
           return false;
         }
       }
+      // Check if the file is a zip file and unzip it.
+      if (options?.checkIfZipFile) {
+        const fileExtension = path.extname(remoteFilePath);
+        if (fileExtension !== ".zip") {
+          throw new Error("File is not a zip file.");
+        }
+      }
       // Read all the file content and create a buffer with 'ascii' encoding.
-      const fileContent = await client.get(remoteFilePath, undefined, {
-        readStreamOptions: { encoding: FILE_DEFAULT_ENCODING },
-      });
+      let fileContent = await client.get(remoteFilePath);
+      if (options?.checkIfZipFile) {
+        const zipFile = new AdmZip(fileContent as Buffer);
+        const [extractedFile] = zipFile.getEntries();
+        fileContent = extractedFile.getData();
+      }
       // Convert the file content to an array of text lines and remove possible blank lines.
       return fileContent
         .toString()

sources/packages/backend/package.json

@@ -62,6 +62,7 @@
     "@nestjs/terminus": "^10.2.3",
     "@nestjs/typeorm": "^10.0.2",
     "@types/ssh2-sftp-client": "^9.0.3",
+    "adm-zip": "^0.5.16",
     "axios": "^1.7.4",
     "bull": "^4.12.2",
     "clamscan": "^2.2.1",
@@ -99,6 +100,7 @@
     "@suites/di.nestjs": "^3.0.0",
     "@suites/doubles.jest": "^3.0.0",
     "@suites/unit": "^3.0.0",
+    "@types/adm-zip": "^0.5.7",
     "@types/clamscan": "^2.0.8",
     "@types/express": "^4.17.8",
     "@types/faker": "^5.1.5",
@@ -177,4 +179,4 @@
       "^@sims/auth(|/.*)$": "<rootDir>/libs/auth/src/$1"
     }
   }
-}
+}