Merge pull request #929 from terascope/new-client-creation

update packages and asset version

.eslintrc

@@ -1,3 +1,6 @@
 {
     "extends": "@terascope",
+    "rules": {
+        "@typescript-eslint/naming-convention": "off"
+    }
 }

.travis.yml

@@ -22,23 +22,16 @@ cache:
 services:
   - docker
 env:
-  - ELASTICSEARCH_VERSION=5.6
   - ELASTICSEARCH_VERSION=6.8.6
   - ELASTICSEARCH_VERSION=7.2.1
 jobs:
   exclude:
     # Don't build es 7 on osx because it is redundant
     - os: osx
       env: ELASTICSEARCH_VERSION=7.2.1
-    # Don't build es 7 on node 12 to reduce the number of test jobs
-    - node_js: '12.22'
-      env: ELASTICSEARCH_VERSION=7.2.1
     # Don't build es 6 on node 12 to reduce the number of test jobs
     - node_js: '12.22'
       env: ELASTICSEARCH_VERSION=6.8.6
-    # Don't build es 5 on node 14 to reduce the number of test jobs
-    - node_js: '14.17'
-      env: ELASTICSEARCH_VERSION=5.6
 before_install:
   # Install latest yarn
   - npm install --global yarn

asset/package.json

@@ -1,6 +1,6 @@
 {
     "name": "asset",
-    "version": "2.7.12",
+    "version": "2.8.0",
     "private": true,
     "workspaces": {
         "nohoist": [
@@ -16,16 +16,16 @@
     "license": "MIT",
     "description": "",
     "dependencies": {
-        "@terascope/data-mate": "^0.37.4",
-        "@terascope/elasticsearch-api": "^3.0.2",
-        "@terascope/elasticsearch-asset-apis": "^0.8.6",
-        "@terascope/job-components": "^0.56.1",
-        "@terascope/teraslice-state-storage": "^0.33.2",
-        "@terascope/utils": "^0.44.1",
+        "@terascope/data-mate": "^0.38.1",
+        "@terascope/elasticsearch-api": "^3.1.1",
+        "@terascope/elasticsearch-asset-apis": "^0.9.0",
+        "@terascope/job-components": "^0.56.4",
+        "@terascope/teraslice-state-storage": "^0.34.1",
+        "@terascope/utils": "^0.44.2",
         "datemath-parser": "^1.0.6",
         "got": "^11.8.3",
         "mocker-data-generator": "^2.12.0",
-        "moment": "^2.29.1"
+        "moment": "^2.29.3"
     },
     "devDependencies": {}
 }

package.json

@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-assets",
     "description": "bundle of processors for teraslice",
-    "version": "2.7.12",
+    "version": "2.8.0",
     "private": true,
     "workspaces": [
         "packages/*",
@@ -18,7 +18,6 @@
         "test:all": "yarn workspaces run test",
         "test": "jest",
         "test:7": "ELASTICSEARCH_VERSION=7.x jest",
-        "test:5": "ELASTICSEARCH_VERSION=5.6 jest",
         "test:watch": "jest --coverage=false --notify --watch --onlyChanged",
         "test:debug": "env DEBUG=\"${DEBUG:-*teraslice*}\" jest --detectOpenHandles --coverage=false --runInBand",
         "publish:changed": "./scripts/publish.sh"
@@ -38,28 +37,28 @@
     },
     "dependencies": {},
     "devDependencies": {
-        "@terascope/data-types": "^0.34.1",
-        "@terascope/elasticsearch-api": "^3.0.2",
-        "@terascope/elasticsearch-asset-apis": "^0.8.5",
+        "@terascope/data-types": "^0.35.1",
+        "@terascope/elasticsearch-api": "^3.1.1",
+        "@terascope/elasticsearch-asset-apis": "^0.9.0",
         "@terascope/eslint-config": "^0.7.1",
-        "@terascope/job-components": "^0.56.1",
-        "@terascope/teraslice-state-storage": "^0.33.2",
-        "@terascope/types": "^0.10.2",
+        "@terascope/job-components": "^0.56.4",
+        "@terascope/teraslice-state-storage": "^0.34.1",
+        "@terascope/types": "^0.10.3",
         "@types/bluebird": "^3.5.36",
         "@types/elasticsearch": "^5.0.40",
         "@types/got": "^9.6.12",
         "@types/jest": "^27.4.1",
-        "@types/node": "^17.0.21",
+        "@types/node": "^17.0.25",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.60.4",
-        "eslint": "^8.10.0",
+        "elasticsearch-store": "^0.61.2",
+        "eslint": "^8.13.0",
         "jest": "^27.5.1",
         "jest-extended": "^2.0.0",
-        "moment": "^2.29.1",
+        "moment": "^2.29.3",
         "nock": "^13.2.4",
         "node-notifier": "^10.0.1",
         "teraslice-test-harness": "^0.28.2",
-        "ts-jest": "^27.1.3",
-        "typescript": "^4.5.5"
+        "ts-jest": "^27.1.4",
+        "typescript": "^4.6.3"
     }
 }

packages/elasticsearch-asset-apis/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@terascope/elasticsearch-asset-apis",
-    "version": "0.8.6",
+    "version": "0.9.0",
     "description": "Elasticsearch reader and sender apis",
     "publishConfig": {
         "access": "public"
@@ -17,21 +17,20 @@
         "test": "jest",
         "test:watch": "jest --coverage=false --notify --watch --onlyChanged",
         "test:debug": "env DEBUG='*teraslice*' jest --detectOpenHandles --coverage=false --runInBand",
-        "test:7": "ELASTICSEARCH_VERSION=7.x jest",
-        "test:5": "ELASTICSEARCH_VERSION=5.6 jest"
+        "test:7": "ELASTICSEARCH_VERSION=7.x jest"
     },
     "homepage": "https://github.com/terascope/elasticsearch-assets",
     "repository": "[email protected]:terascope/elasticsearch-assets.git",
     "author": "Terascope, LLC <[email protected]>",
     "license": "MIT",
     "dependencies": {
-        "@terascope/data-mate": "^0.37.4",
-        "@terascope/data-types": "^0.34.1",
-        "@terascope/elasticsearch-api": "^3.0.2",
-        "@terascope/utils": "^0.44.1",
+        "@terascope/data-mate": "^0.38.1",
+        "@terascope/data-types": "^0.35.1",
+        "@terascope/elasticsearch-api": "^3.1.1",
+        "@terascope/utils": "^0.44.2",
         "datemath-parser": "^1.0.6",
         "got": "^11.8.3",
-        "moment": "^2.29.1"
+        "moment": "^2.29.3"
     },
     "devDependencies": {
         "@types/elasticsearch": "^5.0.40",

packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/ElasticsearchBulkSender.ts

@@ -13,12 +13,12 @@ import {
 export class ElasticsearchBulkSender implements RouteSenderAPI {
     client: elasticAPI.Client;
     config: ElasticsearchSenderConfig;
-    clientVersion: number;
+    isElasticsearch6: boolean;
     private isRouter = false;
 
     constructor(client: elasticAPI.Client, config: ElasticsearchSenderConfig) {
         this.client = client;
-        this.clientVersion = client.getESVersion();
+        this.isElasticsearch6 = client.isElasticsearch6();
         this.config = config;
         // _key is the char from router
         if (config._key && isString(config._key)) this.isRouter = true;
@@ -52,7 +52,7 @@ export class ElasticsearchBulkSender implements RouteSenderAPI {
     }
 
     private getType(): string {
-        if (this.clientVersion < 7 && this.config.type) {
+        if (this.isElasticsearch6 && this.config.type) {
             return this.config.type;
         }
 

packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderClient.ts

@@ -110,12 +110,13 @@ export class ElasticsearchReaderClient implements ReaderClient {
         await this.client.version();
     }
 
-    getSettings(index: string): Promise<SettingResults> {
-        return this._baseClient.indices.getSettings({
+    async getSettings(index: string): Promise<SettingResults> {
+        const results = await this._baseClient.indices.getSettings({
             index,
             flatSettings: true,
             includeDefaults: true,
             allowNoIndices: true
         });
+        return results.body || results;
     }
 }

packages/elasticsearch-asset-apis/test/bulk-send-spec.ts

@@ -2,21 +2,28 @@ import 'jest-extended';
 import { debugLogger, AnyObject, DataEntity } from '@terascope/utils';
 import { WorkerTestHarness } from 'teraslice-test-harness';
 import elasticAPI from '@terascope/elasticsearch-api';
-import { makeClient, cleanupIndex, fetch } from '../test/helpers/elasticsearch';
-import { TEST_INDEX_PREFIX, waitForData } from '../test/helpers';
+import { createClient } from 'elasticsearch-store';
+import { cleanupIndex, fetch } from '../test/helpers/elasticsearch';
+import { TEST_INDEX_PREFIX, waitForData, ELASTICSEARCH_HOST } from '../test/helpers';
 import { createElasticsearchBulkSender } from '../src/elasticsearch-bulk-sender';
 
 describe('elasticsearch bulk sender module', () => {
     const META_ROUTE = 'standard:route';
     const logger = debugLogger('sender_api_test');
-    const client = makeClient();
-    const esClient = elasticAPI(client, logger);
     const senderIndex = `${TEST_INDEX_PREFIX}_sender_api_`;
-    const type = esClient.getESVersion() === 7 ? '_doc' : 'events';
 
+    let apiClient: elasticAPI.Client;
     let harness: WorkerTestHarness;
+    let client: any;
+    let type: string;
 
     beforeAll(async () => {
+        const { client: rawClient } = await createClient({
+            node: ELASTICSEARCH_HOST,
+        } as any, logger);
+        client = rawClient;
+        apiClient = elasticAPI(client, logger);
+        type = apiClient.isElasticsearch6() ? 'events' : '_doc';
         await cleanupIndex(client, `${senderIndex}*`);
     });
 
@@ -39,8 +46,7 @@ describe('elasticsearch bulk sender module', () => {
             },
             config
         ) as any;
-
-        return createElasticsearchBulkSender({ client: esClient, config: senderConfig });
+        return createElasticsearchBulkSender({ client: apiClient, config: senderConfig });
     }
 
     it('can instantiate', async () => {
@@ -70,7 +76,7 @@ describe('elasticsearch bulk sender module', () => {
         it('can format bulk index data with es7', async () => {
             const sender = createSender();
 
-            sender.clientVersion = 7;
+            sender.isElasticsearch6 = false;
 
             const docArray = [DataEntity.make({ action: 'index' })];
 
@@ -357,7 +363,7 @@ describe('elasticsearch bulk sender module', () => {
     });
 
     describe('send', () => {
-        it('can bulk send data', async () => {
+        it('can bulk send data test', async () => {
             const sender = createSender();
             const data = [
                 DataEntity.make({ some: 'data' }),

packages/elasticsearch-asset-apis/test/condensed-data-spec.ts

@@ -2,13 +2,13 @@ import 'jest-extended';
 import { EventEmitter } from 'events';
 import { LATEST_VERSION, DataTypeConfig } from '@terascope/data-types';
 import { debugLogger, DataEntity } from '@terascope/utils';
-import { getESVersion } from 'elasticsearch-store';
+import { createClient } from 'elasticsearch-store';
 import {
     TEST_INDEX_PREFIX,
-    makeClient,
     cleanupIndex,
     populateIndex,
     waitForData,
+    ELASTICSEARCH_HOST
 } from './helpers';
 import condensedData from './fixtures/data/condensed-spread';
 import {
@@ -23,29 +23,33 @@ import {
 } from '../src';
 
 describe('ReaderAPI with condensed time data', () => {
-    const client = makeClient();
-
     const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_api_condensed_`;
     const logger = debugLogger('api-condensed-test');
     const emitter = new EventEmitter();
 
     const condensedIndex = makeIndex(condensedData.index);
     const evenBulkData = condensedData.data.map((obj) => DataEntity.make(obj, { _key: obj.bytes }));
 
-    const readerClient = new ElasticsearchReaderClient(
-        client,
-        { index: condensedIndex },
-        logger,
-    );
-
-    const version = getESVersion(client);
-    const docType = version === 5 ? 'events' : '_doc';
+    const docType = '_doc';
 
     function makeIndex(str: string) {
         return `${readerIndex}_${str}`;
     }
+    let client: any;
+    let readerClient: ElasticsearchReaderClient;
 
     beforeAll(async () => {
+        const { client: esClient, } = await createClient({
+            node: ELASTICSEARCH_HOST,
+        } as any, logger);
+        client = esClient;
+
+        readerClient = new ElasticsearchReaderClient(
+            client,
+            { index: condensedIndex },
+            logger,
+        );
+
         await cleanupIndex(client, makeIndex('*'));
         await populateIndex(client, condensedIndex, condensedData.types, evenBulkData, docType);
         await waitForData(client, condensedIndex, evenBulkData.length);

packages/elasticsearch-asset-apis/test/cross-cluster-spec.ts

@@ -1,12 +1,12 @@
 import 'jest-extended';
-import { getESVersion } from 'elasticsearch-store';
+import { createClient } from 'elasticsearch-store';
 import { LATEST_VERSION, DataTypeConfig } from '@terascope/data-types';
 import { debugLogger, DataEntity } from '@terascope/utils';
 import { DataFrame } from '@terascope/data-mate';
 import { EventEmitter } from 'events';
 import {
     TEST_INDEX_PREFIX,
-    makeClient,
+    ELASTICSEARCH_HOST,
     cleanupIndex,
     populateIndex,
     waitForData,
@@ -22,7 +22,6 @@ import {
 } from '../src';
 
 describe('Reader API', () => {
-    const client = makeClient();
     const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_api_dataframe_cross_cluster`;
     const logger = debugLogger('api-dataFrame-test');
     const emitter = new EventEmitter();
@@ -35,19 +34,23 @@ describe('Reader API', () => {
     }
 
     const crossClusterTestIndex = makeIndex(baseIndex);
-
-    const readerClient = new ElasticsearchReaderClient(
-        client,
-        { index: crossClusterTestIndex },
-        logger,
-    );
-
     const evenBulkData = evenSpread.data.map((obj) => DataEntity.make(obj, { _key: obj.uuid }));
 
-    const version = getESVersion(client);
-    const docType = version === 5 ? 'events' : '_doc';
+    const docType = '_doc';
+    let client: any;
+    let readerClient: ElasticsearchReaderClient;
 
     beforeAll(async () => {
+        const { client: esClient, } = await createClient({
+            node: ELASTICSEARCH_HOST,
+        } as any, logger);
+        client = esClient;
+
+        readerClient = new ElasticsearchReaderClient(
+            client,
+            { index: crossClusterTestIndex },
+            logger,
+        );
         await cleanupIndex(client, makeIndex('*'));
         await populateIndex(client, crossClusterTestIndex, evenSpread.types, evenBulkData, docType);
         await waitForData(client, crossClusterTestIndex, evenBulkData.length);