diff --git a/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts b/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts index 5c2d8b47dd..95fb190f78 100644 --- a/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts +++ b/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts @@ -52,9 +52,9 @@ export class FedRestrictionProcessingService { */ async process(processSummary: ProcessSummary): Promise { const auditUser = this.systemUsersService.systemUser; - // Get the list of all files from SFTP ordered by file name. + // Get the list of all ZIP files from SFTP ordered by file name. const fileSearch = new RegExp( - `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*$`, + `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*\\.(zip|ZIP)$`, "i", ); diff --git a/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts b/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts index 2ce273f35a..7fc56d32a8 100644 --- a/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts +++ b/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts @@ -9,6 +9,7 @@ import { getFileNameAsExtendedCurrentTimestamp, convertToASCII, FILE_DEFAULT_ENCODING, + readFirstExtractedFile, } from "@sims/utilities"; import { LINE_BREAK_SPLIT_REGEX, @@ -154,10 +155,29 @@ export abstract class SFTPIntegrationBase { return false; } } - // Read all the file content and create a buffer with 'ascii' encoding. - const fileContent = await client.get(remoteFilePath, undefined, { - readStreamOptions: { encoding: FILE_DEFAULT_ENCODING }, - }); + let fileContent: string; + const fileExtension = path.extname(remoteFilePath).toLowerCase(); + const isZIPFile = fileExtension === ".zip"; + if (isZIPFile) { + // Read the zip file content with null encoding to avoid data corruption. + const compressedFileContent = (await client.get( + remoteFilePath, + undefined, + { readStreamOptions: { encoding: null } }, + )) as Buffer; + // Read the first file content with 'ascii' encoding. + const { fileName, data } = readFirstExtractedFile( + compressedFileContent, + { encoding: FILE_DEFAULT_ENCODING }, + ); + this.logger.log(`Extracted the first file ${fileName}.`); + fileContent = data; + } else { + // Read all the file content and create a buffer with 'ascii' encoding. + fileContent = (await client.get(remoteFilePath, undefined, { + readStreamOptions: { encoding: FILE_DEFAULT_ENCODING }, + })) as string; + } // Convert the file content to an array of text lines and remove possible blank lines. return fileContent .toString() diff --git a/sources/packages/backend/libs/utilities/src/compressed-file-utils.ts b/sources/packages/backend/libs/utilities/src/compressed-file-utils.ts new file mode 100644 index 0000000000..e24cf1a36b --- /dev/null +++ b/sources/packages/backend/libs/utilities/src/compressed-file-utils.ts @@ -0,0 +1,28 @@ +import * as AdmZip from "adm-zip"; + +/** + * Reads the first extracted file from a compressed archive file. + * @param compressedFileBuffer compressed file buffer. + * @param options options. + * - `encoding`: encoding to read the file. + * @returns first extracted file name and data. + */ +export function readFirstExtractedFile( + compressedFileBuffer: Buffer, + options?: { encoding: string }, +): { + fileName: string; + data: string; +} { + const zipFile = new AdmZip(compressedFileBuffer); + const [firstExtractedFile] = zipFile.getEntries(); + if (!firstExtractedFile) { + throw new Error("No files found in zip file."); + } + // Read the first extracted file with the specified encoding. + const data = zipFile.readAsText(firstExtractedFile, options?.encoding); + return { + fileName: firstExtractedFile.name, + data, + }; +} diff --git a/sources/packages/backend/libs/utilities/src/index.ts b/sources/packages/backend/libs/utilities/src/index.ts index 874e8af4b2..7d1754ebe8 100644 --- a/sources/packages/backend/libs/utilities/src/index.ts +++ b/sources/packages/backend/libs/utilities/src/index.ts @@ -15,3 +15,4 @@ export * from "./math-utils"; export * from "./specialized-string-builder"; export * from "./string-utils"; export * from "./address-utils"; +export * from "./compressed-file-utils"; diff --git a/sources/packages/backend/package-lock.json b/sources/packages/backend/package-lock.json index 8a02b082a5..6e376cb0b5 100644 --- a/sources/packages/backend/package-lock.json +++ b/sources/packages/backend/package-lock.json @@ -29,6 +29,7 @@ "@nestjs/terminus": "^10.2.3", "@nestjs/typeorm": "^10.0.2", "@types/ssh2-sftp-client": "^9.0.3", + "adm-zip": "^0.5.16", "axios": "^1.7.4", "bull": "^4.12.2", "clamscan": "^2.2.1", @@ -66,6 +67,7 @@ "@suites/di.nestjs": "^3.0.0", "@suites/doubles.jest": "^3.0.0", "@suites/unit": "^3.0.0", + "@types/adm-zip": "^0.5.7", "@types/clamscan": "^2.0.8", "@types/express": "^4.17.8", "@types/faker": "^5.1.5", @@ -4653,6 +4655,15 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "devOptional": true }, + "node_modules/@types/adm-zip": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.7.tgz", + "integrity": "sha512-DNEs/QvmyRLurdQPChqq0Md4zGvPwHerAJYWk9l2jCbD1VPpnzRJorOdiq4zsw09NFbYnhfsoEhWtxIzXpn2yw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -5480,6 +5491,14 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "engines": { + "node": ">=12.0" + } + }, "node_modules/ajv": { "version": "8.12.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", diff --git a/sources/packages/backend/package.json b/sources/packages/backend/package.json index 95db168889..13acf35885 100644 --- a/sources/packages/backend/package.json +++ b/sources/packages/backend/package.json @@ -62,6 +62,7 @@ "@nestjs/terminus": "^10.2.3", "@nestjs/typeorm": "^10.0.2", "@types/ssh2-sftp-client": "^9.0.3", + "adm-zip": "^0.5.16", "axios": "^1.7.4", "bull": "^4.12.2", "clamscan": "^2.2.1", @@ -99,6 +100,7 @@ "@suites/di.nestjs": "^3.0.0", "@suites/doubles.jest": "^3.0.0", "@suites/unit": "^3.0.0", + "@types/adm-zip": "^0.5.7", "@types/clamscan": "^2.0.8", "@types/express": "^4.17.8", "@types/faker": "^5.1.5", @@ -177,4 +179,4 @@ "^@sims/auth(|/.*)$": "/libs/auth/src/$1" } } -} \ No newline at end of file +}