diff --git a/asset/asset.json b/asset/asset.json index c7d9432f..1594d352 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,5 +1,5 @@ { "name": "file", - "version": "2.7.2", + "version": "2.7.3", "description": "A set of processors for working with files" } diff --git a/asset/package.json b/asset/package.json index 03823dae..a9166e1b 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,7 +1,7 @@ { "name": "file", "displayName": "Asset", - "version": "2.7.2", + "version": "2.7.3", "private": true, "description": "A set of processors for working with files", "repository": { @@ -20,7 +20,7 @@ "test": "yarn --cwd ../ test" }, "dependencies": { - "@terascope/file-asset-apis": "^0.10.1", + "@terascope/file-asset-apis": "^0.10.2", "@terascope/job-components": "^0.64.0", "csvtojson": "^2.0.10", "fs-extra": "^11.1.1", diff --git a/package.json b/package.json index 99b0fe93..c419f9cd 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "file-assets-bundle", "displayName": "File Assets Bundle", - "version": "2.7.2", + "version": "2.7.3", "private": true, "description": "A set of processors for working with files", "repository": "https://github.com/terascope/file-assets.git", @@ -30,7 +30,7 @@ "dependencies": {}, "devDependencies": { "@terascope/eslint-config": "^0.7.1", - "@terascope/file-asset-apis": "^0.10.1", + "@terascope/file-asset-apis": "^0.10.2", "@terascope/job-components": "^0.64.0", "@terascope/scripts": "0.59.0", "@types/fs-extra": "^11.0.2", diff --git a/packages/file-asset-apis/package.json b/packages/file-asset-apis/package.json index 505b6e9f..11838aa9 100644 --- a/packages/file-asset-apis/package.json +++ b/packages/file-asset-apis/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/file-asset-apis", "displayName": "File Asset Apis", - "version": "0.10.1", + "version": "0.10.2", "description": "file reader and sender apis", "homepage": "https://github.com/terascope/file-assets", "repository": "git@github.com:terascope/file-assets.git", diff --git a/packages/file-asset-apis/src/s3/createS3Client.ts b/packages/file-asset-apis/src/s3/createS3Client.ts index b6de3ba4..243cd619 100644 --- a/packages/file-asset-apis/src/s3/createS3Client.ts +++ b/packages/file-asset-apis/src/s3/createS3Client.ts @@ -1,15 +1,18 @@ import fs from 'fs-extra'; -import { Agent } from 'https'; +import { Agent, AgentOptions as HttpsAgentOptions } from 'https'; import { S3Client as BaseClient } from '@aws-sdk/client-s3'; -import { NodeHttpHandler } from '@aws-sdk/node-http-handler'; +import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler'; import type { S3ClientConfig as baseConfig } from '@aws-sdk/client-s3'; -import { debugLogger, has } from '@terascope/utils'; +import { + debugLogger, has, isEmpty, isNumber +} from '@terascope/utils'; import type { S3Client } from './client-types'; export interface S3ClientConfig extends baseConfig { sslEnabled?: boolean, certLocation?: string, - httpOptions?: object, + httpOptions?: HttpsAgentOptions, + handlerOptions?: Pick, secretAccessKey?: string, accessKeyId?: string } @@ -20,13 +23,17 @@ export async function createS3Client( ): Promise { logger.info(`Using S3 endpoint: ${config.endpoint}`); - // The aws v3 client logs every request and its metadata, it is to intrusive - // should only be used in trace mode otherwise it will log the body request + // The aws v3 client logs every request and its metadata, it is too intrusive + // so should only be used in trace mode otherwise it will log the body request // which has heavy performance implications if (logger.level() === 10) { config.logger = logger; } + let httpOptions = Object.assign( + config.httpOptions || {} + ); + // pull certLocation from env // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-registering-certs.html // Instead of updating the client, we can just update the config before creating the client @@ -42,26 +49,36 @@ export async function createS3Client( // Assumes all certs needed are in a single bundle const certs = await fs.readFile(certPath); - const configHttpOptions = config.httpOptions ?? {}; + httpOptions = { + ...httpOptions, + rejectUnauthorized: true, + ca: [certs] + }; + } - const httpOptions = Object.assign( - { rejectUnauthorized: true }, - configHttpOptions, - { ca: [certs] } - ); + const { connectionTimeout, requestTimeout } = config.handlerOptions || {}; + const requestHandlerOptions = { + ...isNumber(connectionTimeout) && { connectionTimeout }, + ...isNumber(requestTimeout) && { requestTimeout }, + ...!isEmpty(httpOptions) && new Agent(httpOptions) + }; - config.requestHandler = new NodeHttpHandler({ - httpsAgent: new Agent(httpOptions) - }); + if (!isEmpty(requestHandlerOptions)) { + config.requestHandler = new NodeHttpHandler(requestHandlerOptions); } // config specified old style, need to move top level values into credentials if (!has(config, 'credentials') && has(config, 'accessKeyId') && has(config, 'secretAccessKey')) { - const { accessKeyId, secretAccessKey } = config; + const { accessKeyId = '', secretAccessKey = '' } = config; config.credentials = { accessKeyId, secretAccessKey - } as any; + }; + } + + // specified in terafoundation connector config but is now maxAttempts + if (!has(config, 'maxAttempts') && has(config, 'maxRetries')) { + config.maxAttempts = (config as any).maxRetries; } return new BaseClient(config);