#3869 - Federal Restrictions - Unzip ZIP files coming from SFTP

sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts

@@ -52,9 +52,9 @@ export class FedRestrictionProcessingService {
    */
   async process(processSummary: ProcessSummary): Promise<void> {
     const auditUser = this.systemUsersService.systemUser;
-    // Get the list of all files from SFTP ordered by file name.
+    // Get the list of all ZIP files from SFTP ordered by file name.
     const fileSearch = new RegExp(
-      `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*$`,
+      `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*\\.(zip|ZIP)$`,
       "i",
     );
 

sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts

@@ -9,6 +9,7 @@ import {
   getFileNameAsExtendedCurrentTimestamp,
   convertToASCII,
   FILE_DEFAULT_ENCODING,
+  readFirstExtractedFile,
 } from "@sims/utilities";
 import {
   LINE_BREAK_SPLIT_REGEX,
@@ -154,10 +155,29 @@ export abstract class SFTPIntegrationBase<DownloadType> {
           return false;
         }
       }
-      // Read all the file content and create a buffer with 'ascii' encoding.
-      const fileContent = await client.get(remoteFilePath, undefined, {
-        readStreamOptions: { encoding: FILE_DEFAULT_ENCODING },
-      });
+      let fileContent: string | NodeJS.WritableStream | Buffer;
+      const fileExtension = path.parse(remoteFilePath).ext.toLocaleLowerCase();
+      const isZIPFile = fileExtension === ".zip";
+      if (isZIPFile) {
+        // Read the zip file content with null encoding to avoid data corruption.
+        const compressedFileContent = (await client.get(
+          remoteFilePath,
+          undefined,
+          { readStreamOptions: { encoding: null } },
+        )) as Buffer;
+        // Read the first file content with 'ascii' encoding.
+        const { fileName, data } = readFirstExtractedFile(
+          compressedFileContent,
+          { encoding: FILE_DEFAULT_ENCODING },
+        );
+        this.logger.log(`Extracted the first file ${fileName}.`);
+        fileContent = data;
+      } else {
+        // Read all the file content and create a buffer with 'ascii' encoding.
+        fileContent = await client.get(remoteFilePath, undefined, {
+          readStreamOptions: { encoding: FILE_DEFAULT_ENCODING },
+        });
+      }
       // Convert the file content to an array of text lines and remove possible blank lines.
       return fileContent
         .toString()

sources/packages/backend/libs/utilities/src/compressed-file-utils.ts

@@ -0,0 +1,28 @@
+import * as AdmZip from "adm-zip";
+
+/**
+ * Reads the first extracted file from a compressed archive file.
+ * @param compressedFileBuffer
+ * @param options options.
+ * - `encoding`: encoding to read the file.
+ * @returns first extracted file name and data.
+ */
+export function readFirstExtractedFile(
+  compressedFileBuffer: Buffer,
+  options?: { encoding: string },
+): {
+  fileName: string;
+  data: string;
+} {
+  const zipFile = new AdmZip(compressedFileBuffer);
+  const [firstExtractedFile] = zipFile.getEntries();
+  if (!firstExtractedFile) {
+    throw new Error("No files found in zip file");
+  }
+  // Read the first extracted file with the specified encoding.
+  const data = zipFile.readAsText(firstExtractedFile, options?.encoding);
+  return {
+    fileName: firstExtractedFile.name,
+    data,
+  };
+}

sources/packages/backend/libs/utilities/src/index.ts

@@ -15,3 +15,4 @@ export * from "./math-utils";
 export * from "./specialized-string-builder";
 export * from "./string-utils";
 export * from "./address-utils";
+export * from "./compressed-file-utils";

sources/packages/backend/package.json

@@ -62,6 +62,7 @@
     "@nestjs/terminus": "^10.2.3",
     "@nestjs/typeorm": "^10.0.2",
     "@types/ssh2-sftp-client": "^9.0.3",
+    "adm-zip": "^0.5.16",
     "axios": "^1.7.4",
     "bull": "^4.12.2",
     "clamscan": "^2.2.1",
@@ -99,6 +100,7 @@
     "@suites/di.nestjs": "^3.0.0",
     "@suites/doubles.jest": "^3.0.0",
     "@suites/unit": "^3.0.0",
+    "@types/adm-zip": "^0.5.7",
     "@types/clamscan": "^2.0.8",
     "@types/express": "^4.17.8",
     "@types/faker": "^5.1.5",
@@ -177,4 +179,4 @@
       "^@sims/auth(|/.*)$": "<rootDir>/libs/auth/src/$1"
     }
   }
-}
+}