From 6994afc6981a94214f621031f9d28572bd999ab1 Mon Sep 17 00:00:00 2001 From: Shashank Shekhar Date: Wed, 4 Dec 2024 17:11:00 -0800 Subject: [PATCH 1/2] #3869 - Unzip federal restrictions file --- .../fed-restriction.integration.service.ts | 4 ++- .../fed-restriction.processing.service.ts | 2 +- .../src/services/ssh/sftp-integration-base.ts | 36 ++++++++++++++++--- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.integration.service.ts b/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.integration.service.ts index 1dd4afdd89..3d097c8e30 100644 --- a/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.integration.service.ts +++ b/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.integration.service.ts @@ -20,7 +20,9 @@ export class FedRestrictionIntegrationService extends SFTPIntegrationBase< async downloadResponseFile( remoteFilePath: string, ): Promise { - const fileLines = await this.downloadResponseFileLines(remoteFilePath); + const fileLines = await this.downloadResponseFileLines(remoteFilePath, { + checkIfZipFile: true, + }); return fileLines.map((line, index) => { return new FedRestrictionFileRecord(line, index + 1); }); diff --git a/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts b/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts index 388f180315..693fbef7f4 100644 --- a/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts +++ b/sources/packages/backend/libs/integrations/src/esdc-integration/fed-restriction-integration/fed-restriction.processing.service.ts @@ -54,7 +54,7 @@ export class FedRestrictionProcessingService { const auditUser = this.systemUsersService.systemUser; // Get the list of all files from SFTP ordered by file name. const fileSearch = new RegExp( - `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*$`, + `^${this.esdcConfig.environmentCode}CSLS\\.PBC\\.RESTR\\.LIST\\.D[\\w]*\\.[\\d]*.zip$`, "i", ); diff --git a/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts b/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts index 2ce273f35a..0472b56c37 100644 --- a/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts +++ b/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts @@ -1,5 +1,6 @@ import { LoggerService, InjectLogger } from "@sims/utilities/logger"; import { SshService } from "./ssh.service"; +import { unzip } from "node:zlib"; import * as Client from "ssh2-sftp-client"; import * as path from "path"; import { SFTPConfig } from "@sims/utilities/config"; @@ -8,7 +9,6 @@ import { END_OF_LINE, getFileNameAsExtendedCurrentTimestamp, convertToASCII, - FILE_DEFAULT_ENCODING, } from "@sims/utilities"; import { LINE_BREAK_SPLIT_REGEX, @@ -132,6 +132,19 @@ export abstract class SFTPIntegrationBase { options: { checkIfFileExist: boolean }, ): Promise; + /** + * Downloads the file specified on 'fileName' parameter from the + * SFAS integration folder on the SFTP. + * @param remoteFilePath full remote file path with file name. + * @param options download file options. + * - `checkIfZipFile` when set to true, the expectation is to receive a zip file. + * @returns parsed records from the file. + */ + protected async downloadResponseFileLines( + remoteFilePath: string, + options: { checkIfZipFile: boolean }, + ): Promise; + /** * Downloads the file specified on 'fileName' parameter from the * SFAS integration folder on the SFTP. @@ -142,7 +155,7 @@ export abstract class SFTPIntegrationBase { */ protected async downloadResponseFileLines( remoteFilePath: string, - options?: { checkIfFileExist: boolean }, + options?: { checkIfFileExist: boolean; checkIfZipFile: boolean }, ): Promise { this.logger.log(`Downloading file ${remoteFilePath}.`); let client: Client; @@ -154,10 +167,23 @@ export abstract class SFTPIntegrationBase { return false; } } + // Check if the file is a zip file and unzip it. + if (options?.checkIfZipFile) { + const fileExtension = path.extname(remoteFilePath); + if (fileExtension !== ".zip") { + throw new Error("File is not a zip file."); + } + } // Read all the file content and create a buffer with 'ascii' encoding. - const fileContent = await client.get(remoteFilePath, undefined, { - readStreamOptions: { encoding: FILE_DEFAULT_ENCODING }, - }); + let fileContent = await client.get(remoteFilePath); + if (options?.checkIfZipFile) { + unzip(fileContent as Buffer, (err, buffer) => { + if (err) { + throw new Error("Error while unzipping file."); + } + fileContent = buffer; + }); + } // Convert the file content to an array of text lines and remove possible blank lines. return fileContent .toString() From aafc68c3ddf51ea85c59b7cdbd4ac538da2c0010 Mon Sep 17 00:00:00 2001 From: Dheepak Ramanathan Date: Wed, 18 Dec 2024 15:00:01 -0700 Subject: [PATCH 2/2] adding adm zip solution --- .../src/services/ssh/sftp-integration-base.ts | 11 ++++------- sources/packages/backend/package-lock.json | 19 +++++++++++++++++++ sources/packages/backend/package.json | 4 +++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts b/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts index 0472b56c37..d9e857e518 100644 --- a/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts +++ b/sources/packages/backend/libs/integrations/src/services/ssh/sftp-integration-base.ts @@ -1,6 +1,5 @@ import { LoggerService, InjectLogger } from "@sims/utilities/logger"; import { SshService } from "./ssh.service"; -import { unzip } from "node:zlib"; import * as Client from "ssh2-sftp-client"; import * as path from "path"; import { SFTPConfig } from "@sims/utilities/config"; @@ -14,6 +13,7 @@ import { LINE_BREAK_SPLIT_REGEX, SFTP_ARCHIVE_DIRECTORY, } from "@sims/integrations/constants"; +import * as AdmZip from "adm-zip"; /** * Provides the basic features to enable the SFTP integration. @@ -177,12 +177,9 @@ export abstract class SFTPIntegrationBase { // Read all the file content and create a buffer with 'ascii' encoding. let fileContent = await client.get(remoteFilePath); if (options?.checkIfZipFile) { - unzip(fileContent as Buffer, (err, buffer) => { - if (err) { - throw new Error("Error while unzipping file."); - } - fileContent = buffer; - }); + const zipFile = new AdmZip(fileContent as Buffer); + const [extractedFile] = zipFile.getEntries(); + fileContent = extractedFile.getData(); } // Convert the file content to an array of text lines and remove possible blank lines. return fileContent diff --git a/sources/packages/backend/package-lock.json b/sources/packages/backend/package-lock.json index 8a02b082a5..6e376cb0b5 100644 --- a/sources/packages/backend/package-lock.json +++ b/sources/packages/backend/package-lock.json @@ -29,6 +29,7 @@ "@nestjs/terminus": "^10.2.3", "@nestjs/typeorm": "^10.0.2", "@types/ssh2-sftp-client": "^9.0.3", + "adm-zip": "^0.5.16", "axios": "^1.7.4", "bull": "^4.12.2", "clamscan": "^2.2.1", @@ -66,6 +67,7 @@ "@suites/di.nestjs": "^3.0.0", "@suites/doubles.jest": "^3.0.0", "@suites/unit": "^3.0.0", + "@types/adm-zip": "^0.5.7", "@types/clamscan": "^2.0.8", "@types/express": "^4.17.8", "@types/faker": "^5.1.5", @@ -4653,6 +4655,15 @@ "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", "devOptional": true }, + "node_modules/@types/adm-zip": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.7.tgz", + "integrity": "sha512-DNEs/QvmyRLurdQPChqq0Md4zGvPwHerAJYWk9l2jCbD1VPpnzRJorOdiq4zsw09NFbYnhfsoEhWtxIzXpn2yw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -5480,6 +5491,14 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "engines": { + "node": ">=12.0" + } + }, "node_modules/ajv": { "version": "8.12.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", diff --git a/sources/packages/backend/package.json b/sources/packages/backend/package.json index 95db168889..13acf35885 100644 --- a/sources/packages/backend/package.json +++ b/sources/packages/backend/package.json @@ -62,6 +62,7 @@ "@nestjs/terminus": "^10.2.3", "@nestjs/typeorm": "^10.0.2", "@types/ssh2-sftp-client": "^9.0.3", + "adm-zip": "^0.5.16", "axios": "^1.7.4", "bull": "^4.12.2", "clamscan": "^2.2.1", @@ -99,6 +100,7 @@ "@suites/di.nestjs": "^3.0.0", "@suites/doubles.jest": "^3.0.0", "@suites/unit": "^3.0.0", + "@types/adm-zip": "^0.5.7", "@types/clamscan": "^2.0.8", "@types/express": "^4.17.8", "@types/faker": "^5.1.5", @@ -177,4 +179,4 @@ "^@sims/auth(|/.*)$": "/libs/auth/src/$1" } } -} \ No newline at end of file +}