From 1a69308e360b8661b506cac54573fecf4174e9ab Mon Sep 17 00:00:00 2001 From: Lesley Dreyer Date: Mon, 30 Oct 2023 17:14:07 -0700 Subject: [PATCH 1/3] adjust create client for types & to use the httpOpts even if not ssl --- .../file-asset-apis/src/s3/createS3Client.ts | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/packages/file-asset-apis/src/s3/createS3Client.ts b/packages/file-asset-apis/src/s3/createS3Client.ts index b6de3ba4..4dad072f 100644 --- a/packages/file-asset-apis/src/s3/createS3Client.ts +++ b/packages/file-asset-apis/src/s3/createS3Client.ts @@ -1,15 +1,18 @@ import fs from 'fs-extra'; -import { Agent } from 'https'; +import { Agent, AgentOptions as HttpsAgentOptions } from 'https'; import { S3Client as BaseClient } from '@aws-sdk/client-s3'; -import { NodeHttpHandler } from '@aws-sdk/node-http-handler'; +import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler'; import type { S3ClientConfig as baseConfig } from '@aws-sdk/client-s3'; -import { debugLogger, has } from '@terascope/utils'; +import { + debugLogger, has, isEmpty, isNumber +} from '@terascope/utils'; import type { S3Client } from './client-types'; export interface S3ClientConfig extends baseConfig { sslEnabled?: boolean, certLocation?: string, - httpOptions?: object, + httpOptions?: HttpsAgentOptions, + handlerOptions?: Pick, secretAccessKey?: string, accessKeyId?: string } @@ -20,13 +23,17 @@ export async function createS3Client( ): Promise { logger.info(`Using S3 endpoint: ${config.endpoint}`); - // The aws v3 client logs every request and its metadata, it is to intrusive - // should only be used in trace mode otherwise it will log the body request + // The aws v3 client logs every request and its metadata, it is too intrusive + // so should only be used in trace mode otherwise it will log the body request // which has heavy performance implications if (logger.level() === 10) { config.logger = logger; } + let httpOptions = Object.assign( + config.httpOptions || {} + ); + // pull certLocation from env // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-registering-certs.html // Instead of updating the client, we can just update the config before creating the client @@ -42,26 +49,31 @@ export async function createS3Client( // Assumes all certs needed are in a single bundle const certs = await fs.readFile(certPath); - const configHttpOptions = config.httpOptions ?? {}; + httpOptions = { + ...httpOptions, + rejectUnauthorized: true, + ca: [certs] + }; + } - const httpOptions = Object.assign( - { rejectUnauthorized: true }, - configHttpOptions, - { ca: [certs] } - ); + const { connectionTimeout, requestTimeout } = config.handlerOptions || {}; + const requestHandlerOptions = { + ...isNumber(connectionTimeout) && { connectionTimeout }, + ...isNumber(requestTimeout) && { requestTimeout }, + ...!isEmpty(httpOptions) && new Agent(httpOptions) + }; - config.requestHandler = new NodeHttpHandler({ - httpsAgent: new Agent(httpOptions) - }); + if (!isEmpty(requestHandlerOptions)) { + config.requestHandler = new NodeHttpHandler(requestHandlerOptions); } // config specified old style, need to move top level values into credentials if (!has(config, 'credentials') && has(config, 'accessKeyId') && has(config, 'secretAccessKey')) { - const { accessKeyId, secretAccessKey } = config; + const { accessKeyId = '', secretAccessKey = '' } = config; config.credentials = { accessKeyId, secretAccessKey - } as any; + }; } return new BaseClient(config); From 843529d2218f6e63a2d717a8cf64fbe20d8daa9f Mon Sep 17 00:00:00 2001 From: Lesley Dreyer Date: Tue, 31 Oct 2023 09:27:50 -0700 Subject: [PATCH 2/3] convert max retries to max attempts --- packages/file-asset-apis/src/s3/createS3Client.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/file-asset-apis/src/s3/createS3Client.ts b/packages/file-asset-apis/src/s3/createS3Client.ts index 4dad072f..243cd619 100644 --- a/packages/file-asset-apis/src/s3/createS3Client.ts +++ b/packages/file-asset-apis/src/s3/createS3Client.ts @@ -76,5 +76,10 @@ export async function createS3Client( }; } + // specified in terafoundation connector config but is now maxAttempts + if (!has(config, 'maxAttempts') && has(config, 'maxRetries')) { + config.maxAttempts = (config as any).maxRetries; + } + return new BaseClient(config); } From 9a4e975ae6d680280f5b8148c1624c6fbcb52c70 Mon Sep 17 00:00:00 2001 From: Lesley Dreyer Date: Tue, 31 Oct 2023 09:29:52 -0700 Subject: [PATCH 3/3] bump: (patch) @terascope/file-asset-apis@0.10.2, file@2.7.3 bump: (patch) file-assets-bundle@2.7.3 --- asset/asset.json | 2 +- asset/package.json | 4 ++-- package.json | 4 ++-- packages/file-asset-apis/package.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/asset/asset.json b/asset/asset.json index c7d9432f..1594d352 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,5 +1,5 @@ { "name": "file", - "version": "2.7.2", + "version": "2.7.3", "description": "A set of processors for working with files" } diff --git a/asset/package.json b/asset/package.json index 03823dae..a9166e1b 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,7 +1,7 @@ { "name": "file", "displayName": "Asset", - "version": "2.7.2", + "version": "2.7.3", "private": true, "description": "A set of processors for working with files", "repository": { @@ -20,7 +20,7 @@ "test": "yarn --cwd ../ test" }, "dependencies": { - "@terascope/file-asset-apis": "^0.10.1", + "@terascope/file-asset-apis": "^0.10.2", "@terascope/job-components": "^0.64.0", "csvtojson": "^2.0.10", "fs-extra": "^11.1.1", diff --git a/package.json b/package.json index 99b0fe93..c419f9cd 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "file-assets-bundle", "displayName": "File Assets Bundle", - "version": "2.7.2", + "version": "2.7.3", "private": true, "description": "A set of processors for working with files", "repository": "https://github.com/terascope/file-assets.git", @@ -30,7 +30,7 @@ "dependencies": {}, "devDependencies": { "@terascope/eslint-config": "^0.7.1", - "@terascope/file-asset-apis": "^0.10.1", + "@terascope/file-asset-apis": "^0.10.2", "@terascope/job-components": "^0.64.0", "@terascope/scripts": "0.59.0", "@types/fs-extra": "^11.0.2", diff --git a/packages/file-asset-apis/package.json b/packages/file-asset-apis/package.json index 505b6e9f..11838aa9 100644 --- a/packages/file-asset-apis/package.json +++ b/packages/file-asset-apis/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/file-asset-apis", "displayName": "File Asset Apis", - "version": "0.10.1", + "version": "0.10.2", "description": "file reader and sender apis", "homepage": "https://github.com/terascope/file-assets", "repository": "git@github.com:terascope/file-assets.git",