Merge pull request #862 from terascope/expand-s3-client-options

Expand s3 client options

asset/asset.json

@@ -1,5 +1,5 @@
 {
     "name": "file",
-    "version": "2.7.2",
+    "version": "2.7.3",
     "description": "A set of processors for working with files"
 }

asset/package.json

@@ -1,7 +1,7 @@
 {
     "name": "file",
     "displayName": "Asset",
-    "version": "2.7.2",
+    "version": "2.7.3",
     "private": true,
     "description": "A set of processors for working with files",
     "repository": {
@@ -20,7 +20,7 @@
         "test": "yarn --cwd ../ test"
     },
     "dependencies": {
-        "@terascope/file-asset-apis": "^0.10.1",
+        "@terascope/file-asset-apis": "^0.10.2",
         "@terascope/job-components": "^0.64.0",
         "csvtojson": "^2.0.10",
         "fs-extra": "^11.1.1",

package.json

@@ -1,7 +1,7 @@
 {
     "name": "file-assets-bundle",
     "displayName": "File Assets Bundle",
-    "version": "2.7.2",
+    "version": "2.7.3",
     "private": true,
     "description": "A set of processors for working with files",
     "repository": "https://github.com/terascope/file-assets.git",
@@ -30,7 +30,7 @@
     "dependencies": {},
     "devDependencies": {
         "@terascope/eslint-config": "^0.7.1",
-        "@terascope/file-asset-apis": "^0.10.1",
+        "@terascope/file-asset-apis": "^0.10.2",
         "@terascope/job-components": "^0.64.0",
         "@terascope/scripts": "0.59.0",
         "@types/fs-extra": "^11.0.2",

packages/file-asset-apis/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/file-asset-apis",
     "displayName": "File Asset Apis",
-    "version": "0.10.1",
+    "version": "0.10.2",
     "description": "file reader and sender apis",
     "homepage": "https://github.com/terascope/file-assets",
     "repository": "[email protected]:terascope/file-assets.git",

packages/file-asset-apis/src/s3/createS3Client.ts

@@ -1,15 +1,18 @@
 import fs from 'fs-extra';
-import { Agent } from 'https';
+import { Agent, AgentOptions as HttpsAgentOptions } from 'https';
 import { S3Client as BaseClient } from '@aws-sdk/client-s3';
-import { NodeHttpHandler } from '@aws-sdk/node-http-handler';
+import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler';
 import type { S3ClientConfig as baseConfig } from '@aws-sdk/client-s3';
-import { debugLogger, has } from '@terascope/utils';
+import {
+    debugLogger, has, isEmpty, isNumber
+} from '@terascope/utils';
 import type { S3Client } from './client-types';
 
 export interface S3ClientConfig extends baseConfig {
     sslEnabled?: boolean,
     certLocation?: string,
-    httpOptions?: object,
+    httpOptions?: HttpsAgentOptions,
+    handlerOptions?: Pick<NodeHttpHandlerOptions, 'requestTimeout'|'connectionTimeout'>,
     secretAccessKey?: string,
     accessKeyId?: string
 }
@@ -20,13 +23,17 @@ export async function createS3Client(
 ): Promise<S3Client> {
     logger.info(`Using S3 endpoint: ${config.endpoint}`);
 
-    // The aws v3 client logs every request and its metadata, it is to intrusive
-    // should only be used in trace mode otherwise it will log the body request
+    // The aws v3 client logs every request and its metadata, it is too intrusive
+    // so should only be used in trace mode otherwise it will log the body request
     // which has heavy performance implications
     if (logger.level() === 10) {
         config.logger = logger;
     }
 
+    let httpOptions = Object.assign(
+        config.httpOptions || {}
+    );
+
     // pull certLocation from env
     // https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/node-registering-certs.html
     // Instead of updating the client, we can just update the config before creating the client
@@ -42,26 +49,36 @@ export async function createS3Client(
         // Assumes all certs needed are in a single bundle
         const certs = await fs.readFile(certPath);
 
-        const configHttpOptions = config.httpOptions ?? {};
+        httpOptions = {
+            ...httpOptions,
+            rejectUnauthorized: true,
+            ca: [certs]
+        };
+    }
 
-        const httpOptions = Object.assign(
-            { rejectUnauthorized: true },
-            configHttpOptions,
-            { ca: [certs] }
-        );
+    const { connectionTimeout, requestTimeout } = config.handlerOptions || {};
+    const requestHandlerOptions = {
+        ...isNumber(connectionTimeout) && { connectionTimeout },
+        ...isNumber(requestTimeout) && { requestTimeout },
+        ...!isEmpty(httpOptions) && new Agent(httpOptions)
+    };
 
-        config.requestHandler = new NodeHttpHandler({
-            httpsAgent: new Agent(httpOptions)
-        });
+    if (!isEmpty(requestHandlerOptions)) {
+        config.requestHandler = new NodeHttpHandler(requestHandlerOptions);
     }
 
     // config specified old style, need to move top level values into credentials
     if (!has(config, 'credentials') && has(config, 'accessKeyId') && has(config, 'secretAccessKey')) {
-        const { accessKeyId, secretAccessKey } = config;
+        const { accessKeyId = '', secretAccessKey = '' } = config;
         config.credentials = {
             accessKeyId,
             secretAccessKey
-        } as any;
+        };
+    }
+
+    // specified in terafoundation connector config but is now maxAttempts
+    if (!has(config, 'maxAttempts') && has(config, 'maxRetries')) {
+        config.maxAttempts = (config as any).maxRetries;
     }
 
     return new BaseClient(config);