diff --git a/.eslintrc b/.eslintrc index 015e74f85..728df68c5 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,6 +1,14 @@ { "extends": "@terascope", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, "rules": { - "@typescript-eslint/naming-convention": "off" - } + "@typescript-eslint/naming-convention": "off", + "@typescript-eslint/no-duplicate-enum-values": "warn", + "import/extensions": "off", + "import/no-import-module-exports": "off" + }, + "ignorePatterns":[] } diff --git a/.github/workflows/test-asset.yml b/.github/workflows/test-asset.yml index e2c883139..5954e0f18 100644 --- a/.github/workflows/test-asset.yml +++ b/.github/workflows/test-asset.yml @@ -8,34 +8,6 @@ on: branches: [ master ] jobs: - test-es6-legacy-linux: - runs-on: ubuntu-latest - env: - NODE_OPTIONS: "--max-old-space-size=4096" - strategy: - matrix: - # NOTE: Hard Coded Node Version array, should match array in build-and-publish-asset.yml - node-version: [18.19.1, 20.11.1, 22.2.0] - steps: - - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'yarn' - # we login to docker to avoid docker pull limit rates - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - run: yarn setup - - run: yarn lint - - run: yarn --silent test:legacy - test-elasticsearch-assets: runs-on: ubuntu-latest env: diff --git a/asset/asset.json b/asset/asset.json index 2f7bfef8d..5a62768aa 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,4 +1,4 @@ { "name": "elasticsearch", - "version": "3.7.2" + "version": "4.0.0" } diff --git a/asset/package.json b/asset/package.json index 177fdc344..07dab48e4 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,11 +1,12 @@ { "name": "asset", "displayName": "Asset", - "version": "3.7.2", + "version": "4.0.0", "private": true, "description": "", "license": "MIT", "author": "", + "type": "module", "workspaces": { "nohoist": [ "**" @@ -17,15 +18,14 @@ "test": "yarn --cwd ../ test" }, "dependencies": { - "@terascope/data-mate": "^0.56.2", - "@terascope/elasticsearch-api": "^3.20.2", - "@terascope/elasticsearch-asset-apis": "^0.12.1", - "@terascope/job-components": "^0.75.1", - "@terascope/teraslice-state-storage": "^0.53.2", - "@terascope/utils": "^0.59.1", + "@terascope/data-mate": "^0.56.3", + "@terascope/elasticsearch-api": "^3.20.3", + "@terascope/elasticsearch-asset-apis": "^1.0.0", + "@terascope/job-components": "^1.0.1", + "@terascope/teraslice-state-storage": "^0.53.3", + "@terascope/utils": "^0.59.3", "datemath-parser": "^1.0.6", - "got": "^11.8.3", - "mocker-data-generator": "^2.12.0", + "got": "^13.0.0", "moment": "^2.30.1" }, "devDependencies": {}, diff --git a/asset/src/__lib/DateReaderAPISlicer.ts b/asset/src/__lib/DateReaderAPISlicer.ts index 0dc0e4e29..9ea574740 100644 --- a/asset/src/__lib/DateReaderAPISlicer.ts +++ b/asset/src/__lib/DateReaderAPISlicer.ts @@ -6,8 +6,8 @@ import moment from 'moment'; import { ElasticsearchReaderAPI, DateSlicerRanges } from '@terascope/elasticsearch-asset-apis'; -import { ESDateConfig } from '../elasticsearch_reader/interfaces'; -import { ElasticReaderFactoryAPI } from '../elasticsearch_reader_api/interfaces'; +import { ESDateConfig } from '../elasticsearch_reader/interfaces.js'; +import { ElasticReaderFactoryAPI } from '../elasticsearch_reader_api/interfaces.js'; export class DateReaderAPISlicer extends ParallelSlicer { protected api!: ElasticsearchReaderAPI; diff --git a/asset/src/__lib/ReaderAPIFetcher.ts b/asset/src/__lib/ReaderAPIFetcher.ts index a22ef0568..0aa73fcc9 100644 --- a/asset/src/__lib/ReaderAPIFetcher.ts +++ b/asset/src/__lib/ReaderAPIFetcher.ts @@ -1,8 +1,8 @@ import { Fetcher, DataEntity, isPromAvailable } from '@terascope/job-components'; import { DataFrame } from '@terascope/data-mate'; import { ElasticsearchReaderAPI, ReaderSlice } from '@terascope/elasticsearch-asset-apis'; -import { ESDateConfig } from '../elasticsearch_reader/interfaces'; -import { ElasticReaderFactoryAPI } from '../elasticsearch_reader_api/interfaces'; +import { ESDateConfig } from '../elasticsearch_reader/interfaces.js'; +import { ElasticReaderFactoryAPI } from '../elasticsearch_reader_api/interfaces.js'; export class ReaderAPIFetcher extends Fetcher { api!: ElasticsearchReaderAPI; diff --git a/asset/src/elasticsearch_bulk/interfaces.ts b/asset/src/elasticsearch_bulk/interfaces.ts index c6352d972..930ee4375 100644 --- a/asset/src/elasticsearch_bulk/interfaces.ts +++ b/asset/src/elasticsearch_bulk/interfaces.ts @@ -1,5 +1,5 @@ import { OpConfig } from '@terascope/job-components'; -import { ElasticsearchAPISenderConfig } from '../elasticsearch_sender_api/interfaces'; +import { ElasticsearchAPISenderConfig } from '../elasticsearch_sender_api/interfaces.js'; export interface ElasticsearchBulkConfig extends ElasticsearchAPISenderConfig, OpConfig { api_name: string; diff --git a/asset/src/elasticsearch_bulk/processor.ts b/asset/src/elasticsearch_bulk/processor.ts index 5f7e01854..e048debc1 100644 --- a/asset/src/elasticsearch_bulk/processor.ts +++ b/asset/src/elasticsearch_bulk/processor.ts @@ -1,8 +1,8 @@ import { BatchProcessor, isPromAvailable } from '@terascope/job-components'; import { DataEntity } from '@terascope/utils'; import { ElasticsearchBulkSender } from '@terascope/elasticsearch-asset-apis'; -import { ElasticSenderAPI } from '../elasticsearch_sender_api/interfaces'; -import { ElasticsearchBulkConfig } from './interfaces'; +import { ElasticSenderAPI } from '../elasticsearch_sender_api/interfaces.js'; +import { ElasticsearchBulkConfig } from './interfaces.js'; export default class ElasticsearchBulk extends BatchProcessor { client!: ElasticsearchBulkSender; diff --git a/asset/src/elasticsearch_bulk/schema.ts b/asset/src/elasticsearch_bulk/schema.ts index abfc93ab6..3bde37322 100644 --- a/asset/src/elasticsearch_bulk/schema.ts +++ b/asset/src/elasticsearch_bulk/schema.ts @@ -3,8 +3,8 @@ import { AnyObject, isString, getTypeOf, cloneDeep, isNumber, isNotNil } from '@terascope/job-components'; -import { ElasticsearchBulkConfig } from './interfaces'; -import { DEFAULT_API_NAME } from '../elasticsearch_sender_api/interfaces'; +import { ElasticsearchBulkConfig } from './interfaces.js'; +import { DEFAULT_API_NAME } from '../elasticsearch_sender_api/interfaces.js'; export const schema: AnyObject = { size: { diff --git a/asset/src/elasticsearch_data_generator/counter.ts b/asset/src/elasticsearch_data_generator/counter.ts deleted file mode 100644 index 6c8c2c0eb..000000000 --- a/asset/src/elasticsearch_data_generator/counter.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { CounterResults } from './interfaces'; - -export default class Counter { - isPersistent: boolean; - numOfRecordsProcessed: number; - sliceSize: number; - size: number; - handle: () => Promise; - - constructor(isPersistent: boolean, size: number, sliceSize = 5000, startingCount = 0) { - this.numOfRecordsProcessed = startingCount; - this.sliceSize = sliceSize; - this.size = size; - this.isPersistent = isPersistent; - - this.handle = async () => { - let count = this.sliceSize; - - if (!isPersistent) { - if (this.numOfRecordsProcessed >= this.size) { - return null; - } - - if (this.numOfRecordsProcessed + this.sliceSize >= this.size) { - count = this.size - this.numOfRecordsProcessed; - } - } - - this.numOfRecordsProcessed += count; - - return { count, processed: this.numOfRecordsProcessed }; - }; - } -} diff --git a/asset/src/elasticsearch_data_generator/data-schema.ts b/asset/src/elasticsearch_data_generator/data-schema.ts deleted file mode 100644 index 24f25766c..000000000 --- a/asset/src/elasticsearch_data_generator/data-schema.ts +++ /dev/null @@ -1,113 +0,0 @@ -import moment from 'moment'; -import { AnyObject, isEmpty } from '@terascope/utils'; -import { dateFormat } from '@terascope/elasticsearch-asset-apis'; -import { DataGenerator, DateOptions, IDType } from './interfaces'; - -function regexID(type: IDType) { - const reg = { randexp: '' }; - - if (type === 'base64url') { - // eslint-disable-next-line - reg.randexp = '[a-zA-Z1-9\-\_]\w{8}'; - } - if (type === 'hexadecimal') { - reg.randexp = '[0-9a-f]{8}'; - } - if (type === 'HEXADECIMAL') { - reg.randexp = '[0-9A-F]{8}'; - } - - return reg; -} - -function utcDate() { - return new Date().toISOString(); -} - -function dateNow() { - return moment().format(dateFormat); -} - -function isoBetween(start: number, diff: number) { - // ex. "2016-01-19T13:48:08.426-07:00" - return () => moment(start + (Math.random() * diff)).format(dateFormat); -} - -function utcBetween(start: number, diff: number) { - // ex. "2016-01-19T20:48:08.426Z" , compare to isoBetween, same dates - return () => new Date(start + (Math.random() * diff)).toISOString(); -} - -const formatOptions = { - dateNow, - isoBetween, - utcDate, - utcBetween -}; - -const nativeSchema = { - ip: { - faker: 'internet.ip' - }, - userAgent: { - faker: 'internet.userAgent' - }, - url: { - faker: 'internet.url' - }, - uuid: { - faker: 'datatype.uuid' - }, - created: { - function: dateNow - }, - ipv6: { - chance: 'ipv6' - }, - location: { - chance: 'coordinates' - }, - bytes: { - chance: 'integer({"min": 7850, "max": 5642867})' - } -}; - -interface FormatOptions { - start?: number; - end?: number; - diff?: number; -} - -function getFormatFunction(format: DateOptions, options: FormatOptions = {}) { - const { start, diff } = options; - - if (format === DateOptions.isoBetween || format === DateOptions.utcBetween) { - return formatOptions[format](start as number, diff as number); - } - - return formatOptions[format]; -} - -export default function getSchema(opConfig: DataGenerator, otherSchema: AnyObject): AnyObject { - const startDate = opConfig.start ? moment(opConfig.start) : moment(0); // 01 January, 1970 UTC - const endDate = opConfig.end ? moment(opConfig.end) : moment(); - const schema: AnyObject = isEmpty(otherSchema) ? nativeSchema : otherSchema; - const start = startDate.valueOf(); - const end = endDate.valueOf(); - const diff = end - start; - - if (opConfig.format) { - schema[opConfig.date_key].function = getFormatFunction(opConfig.format, { start, diff }); - } - - if (opConfig.set_id) { - schema.id = regexID(opConfig.set_id); - } - - if (opConfig.id_start_key) { - const reg = schema.id.randexp; - schema.id.randexp = `${opConfig.id_start_key}${reg}`; - } - - return schema; -} diff --git a/asset/src/elasticsearch_data_generator/fetcher.ts b/asset/src/elasticsearch_data_generator/fetcher.ts deleted file mode 100644 index 1b7ba9fce..000000000 --- a/asset/src/elasticsearch_data_generator/fetcher.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { - Fetcher, WorkerContext, ExecutionConfig, TSError, AnyObject -} from '@terascope/job-components'; -import mocker from 'mocker-data-generator'; -import path from 'path'; -import { existsSync } from 'fs'; -import { deprecate } from 'util'; -import { DataGenerator, CounterResults } from './interfaces'; -import defaultSchema from './data-schema'; - -const isDeprecated = deprecate(() => {}, 'elasticsearch_data_generator is deprecated, please use data_generator from standard assets'); -export default class DataGeneratorFetcher extends Fetcher { - dataSchema: any; - - constructor(context: WorkerContext, opConfig: DataGenerator, exConfig: ExecutionConfig) { - super(context, opConfig, exConfig); - this.dataSchema = parsedSchema(opConfig); - isDeprecated(); - } - - async fetch(slice?: CounterResults): Promise { - if (slice == null) return []; - const { count } = slice; - - if (this.opConfig.stress_test) { - return mocker() - .schema('schema', this.dataSchema, 1) - .build() - .then((dataObj) => { - const results = []; - const data = dataObj.schema[0]; - for (let i = 0; i < count; i += 1) { - results.push(data); - } - return results; - }) - .catch((err) => Promise.reject(new TSError(err, { reason: 'could not generate mocked data' }))); - } - - return mocker() - .schema('schema', this.dataSchema, count) - .build() - .then((dataObj) => dataObj.schema) - .catch((err) => Promise.reject(new TSError(err, { reason: 'could not generate mocked data' }))); - } -} - -function parsedSchema(opConfig: DataGenerator) { - let dataSchema = {}; - - if (opConfig.json_schema) { - const firstPath = opConfig.json_schema; - const nextPath = path.join(process.cwd(), opConfig.json_schema); - - try { - if (existsSync(firstPath)) { - dataSchema = require(firstPath); - } else { - dataSchema = require(nextPath); - } - return dataSchema; - } catch (err) { - throw new TSError(err, { reason: `Could not retrieve code for: ${opConfig._op}` }); - } - } else { - return defaultSchema(opConfig, dataSchema); - } -} diff --git a/asset/src/elasticsearch_data_generator/interfaces.ts b/asset/src/elasticsearch_data_generator/interfaces.ts deleted file mode 100644 index 552d21a3d..000000000 --- a/asset/src/elasticsearch_data_generator/interfaces.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { OpConfig } from '@terascope/job-components'; - -export enum DateOptions { - dateNow = 'dateNow', - isoBetween = 'isoBetween', - utcDate = 'utcDate', - utcBetween = 'utcBetween' -} - -export enum IDType { - base64url = 'base64url', - hexadecimal = 'hexadecimal', - HEXADECIMAL = 'HEXADECIMAL', -} - -export interface DataGenerator extends OpConfig { - json_schema?: string; - size: number; - start?: string | number; - end?: string | number; - format?: DateOptions; - stress_test: boolean; - date_key: string; - set_id?: IDType; - id_start_key?: string; -} - -export type CounterResults = null | { count: number; processed: number } diff --git a/asset/src/elasticsearch_data_generator/schema.ts b/asset/src/elasticsearch_data_generator/schema.ts deleted file mode 100644 index 5597fbdbe..000000000 --- a/asset/src/elasticsearch_data_generator/schema.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { - ConvictSchema, ValidatedJobConfig, getOpConfig, AnyObject -} from '@terascope/job-components'; -import { DataGenerator } from './interfaces'; - -export default class Schema extends ConvictSchema { - validateJob(job: ValidatedJobConfig): void { - const opConfig = getOpConfig(job, 'elasticsearch_data_generator'); - if (!opConfig) throw new Error('No opConfig was found for operation elasticsearch_data_generator on the job'); - - if (opConfig.id_start_key && !opConfig.set_id) { - throw new Error('Invalid elasticsearch_data_generator configuration, id_start_key must be used with set_id parameter, please set the missing parameters'); - } - - if (opConfig.set_id) { - const indexSelectorConfig = getOpConfig(job, 'elasticsearch_index_selector'); - if (!indexSelectorConfig) throw new Error('No opConfig was found for operation elasticsearch_index_selector on the job'); - - if (!indexSelectorConfig.id_field) { - throw new Error('Invalid elasticsearch_data_generator configuration, set_id must be used in tandem with id_field which is set in elasticsearch_index_selector'); - } - - if (indexSelectorConfig.id_field !== 'id') { - throw new Error('Invalid elasticsearch_data_generator configuration, id_field must be set to "id" when elasticsearch_data_generator is creating ids'); - } - } - } - - build(): AnyObject { - return { - json_schema: { - doc: 'file path to custom data schema', - default: '', - format: 'optional_String' - }, - size: { - doc: 'The limit to the number of docs pulled in a chunk, if the number of docs retrieved ' - + 'by the interval exceeds this number, it will cause the function to recurse to provide a smaller batch', - default: 5000, - format(val: any) { - if (isNaN(val)) { - throw new Error('Invalid size parameter for elasticsearch_data_generator, must be a number'); - } else if (val <= 0) { - throw new Error('Invalid size parameter for elasticsearch_data_generator, must be greater than zero'); - } - } - }, - start: { - doc: 'The start date (ISOString or in ms) to which it will read from ', - default: '', - format: 'optional_Date' - }, - end: { - doc: 'The end date (ISOString or in ms) to which it will read to', - default: '', - format: 'optional_Date' - }, - format: { - doc: 'This is only used with the teraslice provided schema, can elect different time structures' - + 'such as dateNow, utcDate, utcBetween and isoBetween', - default: '', - format: 'optional_String' - }, - stress_test: { - doc: 'used to speed up the creation process to test elasticsearch load', - default: false, - format: Boolean - }, - date_key: { - doc: 'key value on schema where date should reside', - default: 'created', - format: String - }, - set_id: { - doc: 'used to make an id on the data that will be used for the doc _id for elasticsearch, values: base64url, hexadecimal, HEXADECIMAL', - default: '', - format: 'optional_String' - }, - id_start_key: { - doc: 'set if you would like to force the first part of the ID to a certain character', - default: '', - format: 'optional_String' - } - }; - } -} diff --git a/asset/src/elasticsearch_data_generator/slicer.ts b/asset/src/elasticsearch_data_generator/slicer.ts deleted file mode 100644 index 6ce9f934f..000000000 --- a/asset/src/elasticsearch_data_generator/slicer.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Slicer, get, SlicerRecoveryData } from '@terascope/job-components'; -import { DataGenerator, CounterResults } from './interfaces'; -import Counter from './counter'; - -export default class DataGeneratorSlicer extends Slicer { - countHandle!: () => Promise; - - async initialize(recoveryData: SlicerRecoveryData[]): Promise { - await super.initialize(recoveryData); - - const { size } = this.opConfig; - const isPersistent = this.executionConfig.lifecycle === 'persistent'; - let alreadyProcessed: undefined|number; - - if (this.recoveryData) { - alreadyProcessed = get(this.recoveryData[0], 'lastSlice.processed', 0); - } - - const opListSize = this.executionConfig.operations.length - 1; - const lastOp = this.executionConfig.operations[opListSize]; - - const counter = new Counter(isPersistent, size, lastOp.size, alreadyProcessed); - this.countHandle = counter.handle; - } - - maxQueueLength(): number { - return this.workersConnected * 3; - } - - isRecoverable(): boolean { - return true; - } - - async slice(): Promise { - return this.countHandle(); - } -} diff --git a/asset/src/elasticsearch_reader/fetcher.ts b/asset/src/elasticsearch_reader/fetcher.ts index 8ef243bec..c30000f37 100644 --- a/asset/src/elasticsearch_reader/fetcher.ts +++ b/asset/src/elasticsearch_reader/fetcher.ts @@ -1,4 +1,4 @@ -import { ReaderAPIFetcher } from '../__lib/ReaderAPIFetcher'; +import { ReaderAPIFetcher } from '../__lib/ReaderAPIFetcher.js'; export default class ElasticsearchDateFetcher extends ReaderAPIFetcher { diff --git a/asset/src/elasticsearch_reader/schema.ts b/asset/src/elasticsearch_reader/schema.ts index 91625f699..792eb5c87 100644 --- a/asset/src/elasticsearch_reader/schema.ts +++ b/asset/src/elasticsearch_reader/schema.ts @@ -2,9 +2,9 @@ import { ConvictSchema, ValidatedJobConfig, AnyObject, getTypeOf, isNil } from '@terascope/job-components'; -import { opSchema } from '../__lib/schema'; -import { ESReaderConfig } from './interfaces'; -import { DEFAULT_API_NAME } from '../elasticsearch_reader_api/interfaces'; +import { opSchema } from '../__lib/schema.js'; +import { ESReaderConfig } from './interfaces.js'; +import { DEFAULT_API_NAME } from '../elasticsearch_reader_api/interfaces.js'; export default class Schema extends ConvictSchema { validateJob(job: ValidatedJobConfig): void { diff --git a/asset/src/elasticsearch_reader/slicer.ts b/asset/src/elasticsearch_reader/slicer.ts index 68f35291e..09df49154 100644 --- a/asset/src/elasticsearch_reader/slicer.ts +++ b/asset/src/elasticsearch_reader/slicer.ts @@ -1,4 +1,4 @@ -import { DateReaderAPISlicer } from '../__lib/DateReaderAPISlicer'; +import { DateReaderAPISlicer } from '../__lib/DateReaderAPISlicer.js'; export default class ElasticsearchDateSlicer extends DateReaderAPISlicer { diff --git a/asset/src/elasticsearch_reader_api/schema.ts b/asset/src/elasticsearch_reader_api/schema.ts index 33ac7cb8e..90d76acca 100644 --- a/asset/src/elasticsearch_reader_api/schema.ts +++ b/asset/src/elasticsearch_reader_api/schema.ts @@ -8,8 +8,8 @@ import moment from 'moment'; // @ts-expect-error import dateMath from 'datemath-parser'; import { dateOptions, IDType } from '@terascope/elasticsearch-asset-apis'; -import { ElasticsearchReaderAPIConfig, DEFAULT_API_NAME } from './interfaces'; -import { isValidIndex } from '../__lib/schema'; +import { ElasticsearchReaderAPIConfig, DEFAULT_API_NAME } from './interfaces.js'; +import { isValidIndex } from '../__lib/schema.js'; export const schema = { index: { diff --git a/asset/src/elasticsearch_sender_api/api.ts b/asset/src/elasticsearch_sender_api/api.ts index ad89679b5..982f341d9 100644 --- a/asset/src/elasticsearch_sender_api/api.ts +++ b/asset/src/elasticsearch_sender_api/api.ts @@ -8,7 +8,7 @@ import { createElasticsearchBulkSender, ElasticsearchBulkSender } from '@terascope/elasticsearch-asset-apis'; -import { ElasticsearchAPISenderConfig } from './interfaces'; +import { ElasticsearchAPISenderConfig } from './interfaces.js'; export default class ElasticsearchSenderAPI extends APIFactory { diff --git a/asset/src/elasticsearch_sender_api/schema.ts b/asset/src/elasticsearch_sender_api/schema.ts index 005ae2817..57a2c4df7 100644 --- a/asset/src/elasticsearch_sender_api/schema.ts +++ b/asset/src/elasticsearch_sender_api/schema.ts @@ -2,9 +2,9 @@ import { ConvictSchema, AnyObject, cloneDeep, ValidatedJobConfig, APIConfig, getOpConfig, } from '@terascope/job-components'; -import { ElasticsearchAPISenderConfig, DEFAULT_API_NAME } from './interfaces'; -import { isValidIndex } from '../__lib/schema'; -import { schema } from '../elasticsearch_bulk/schema'; +import { ElasticsearchAPISenderConfig, DEFAULT_API_NAME } from './interfaces.js'; +import { isValidIndex } from '../__lib/schema.js'; +import { schema } from '../elasticsearch_bulk/schema.js'; const newSchema: AnyObject = cloneDeep(schema); diff --git a/asset/src/elasticsearch_state_storage/schema.ts b/asset/src/elasticsearch_state_storage/schema.ts index 69f36e016..8cb76c830 100644 --- a/asset/src/elasticsearch_state_storage/schema.ts +++ b/asset/src/elasticsearch_state_storage/schema.ts @@ -2,7 +2,7 @@ import { ConvictSchema, AnyObject, isNumber, getTypeOf, isString } from '@terascope/job-components'; -import { ESStateStorageConfig } from './interfaces'; +import { ESStateStorageConfig } from './interfaces.js'; export default class Schema extends ConvictSchema { build(): AnyObject { diff --git a/asset/src/id_reader/fetcher.ts b/asset/src/id_reader/fetcher.ts index fd25ce088..0d1d8c72f 100644 --- a/asset/src/id_reader/fetcher.ts +++ b/asset/src/id_reader/fetcher.ts @@ -1,4 +1,4 @@ -import { ReaderAPIFetcher } from '../__lib/ReaderAPIFetcher'; +import { ReaderAPIFetcher } from '../__lib/ReaderAPIFetcher.js'; export default class ElasticsearchIDFetcher extends ReaderAPIFetcher { diff --git a/asset/src/id_reader/schema.ts b/asset/src/id_reader/schema.ts index ebc17d554..dfc58b15b 100644 --- a/asset/src/id_reader/schema.ts +++ b/asset/src/id_reader/schema.ts @@ -1,7 +1,7 @@ import { ConvictSchema, ValidatedJobConfig, AnyObject } from '@terascope/job-components'; -import { opSchema } from '../__lib/schema'; -import { ESIDReaderConfig } from './interfaces'; -import { DEFAULT_API_NAME } from '../elasticsearch_reader_api/interfaces'; +import { opSchema } from '../__lib/schema.js'; +import { ESIDReaderConfig } from './interfaces.js'; +import { DEFAULT_API_NAME } from '../elasticsearch_reader_api/interfaces.js'; export default class Schema extends ConvictSchema { validateJob(job: ValidatedJobConfig): void { diff --git a/asset/src/id_reader/slicer.ts b/asset/src/id_reader/slicer.ts index d7f71047b..3d90d6ee4 100644 --- a/asset/src/id_reader/slicer.ts +++ b/asset/src/id_reader/slicer.ts @@ -2,8 +2,8 @@ import { ParallelSlicer, SlicerFn, SlicerRecoveryData } from '@terascope/job-components'; import { ElasticsearchReaderAPI, IDSlicerRanges } from '@terascope/elasticsearch-asset-apis'; -import { ESIDReaderConfig } from './interfaces'; -import { ElasticReaderFactoryAPI, ElasticsearchReaderAPIConfig } from '../elasticsearch_reader_api/interfaces'; +import { ESIDReaderConfig } from './interfaces.js'; +import { ElasticReaderFactoryAPI, ElasticsearchReaderAPIConfig } from '../elasticsearch_reader_api/interfaces.js'; export default class ESIDSlicer extends ParallelSlicer { api!: ElasticsearchReaderAPI; diff --git a/asset/src/index.ts b/asset/src/index.ts index 605d78d6f..69bf89489 100644 --- a/asset/src/index.ts +++ b/asset/src/index.ts @@ -1,36 +1,29 @@ -// TODO: types are wrong for AssetRepository -// import { AssetRepository } from '@terascope/job-components'; +import ESBulkSender from './elasticsearch_bulk/processor.js'; +import ESBulkSenderSchema from './elasticsearch_bulk/schema.js'; -import ESBulkSender from '../src/elasticsearch_bulk/processor'; -import ESBulkSenderSchema from '../src/elasticsearch_bulk/schema'; +import ESDateFetcher from './elasticsearch_reader/fetcher.js'; +import ESDateSlicer from './elasticsearch_reader/slicer.js'; +import ESDateSchema from './elasticsearch_reader/schema.js'; -import ESDataGenFetcher from '../src/elasticsearch_data_generator/fetcher'; -import ESDataGenSlicer from '../src/elasticsearch_data_generator/slicer'; -import ESDataGenSchema from '../src/elasticsearch_data_generator/schema'; +import ESReaderAPI from './elasticsearch_reader_api/api.js'; +import ESReaderAPISchema from './elasticsearch_reader_api/schema.js'; -import ESDateFetcher from '../src/elasticsearch_reader/fetcher'; -import ESDateSlicer from '../src/elasticsearch_reader/slicer'; -import ESDateSchema from '../src/elasticsearch_reader/schema'; +import ESSenderAPI from './elasticsearch_sender_api/api.js'; +import ESSenderAPISchema from './elasticsearch_sender_api/schema.js'; -import ESReaderAPI from '../src/elasticsearch_reader_api/api'; -import ESReaderAPISchema from '../src/elasticsearch_reader_api/schema'; +import SpacesReaderAPI from './spaces_reader_api/api.js'; +import SpacesReaderAPISchema from './spaces_reader_api/schema.js'; -import ESSenderAPI from '../src/elasticsearch_sender_api/api'; -import ESSenderAPISchema from '../src/elasticsearch_sender_api/schema'; +import SpacesFetcher from './spaces_reader/fetcher.js'; +import SpacesSlicer from './spaces_reader/slicer.js'; +import SpacesSchema from './spaces_reader/schema.js'; -import SpacesReaderAPI from '../src/spaces_reader_api/api'; -import SpacesReaderAPISchema from '../src/spaces_reader_api/schema'; +import IDFetcher from './id_reader/fetcher.js'; +import IDSlicer from './id_reader/slicer.js'; +import IDSchema from './id_reader/schema.js'; -import SpacesFetcher from '../src/spaces_reader/fetcher'; -import SpacesSlicer from '../src/spaces_reader/slicer'; -import SpacesSchema from '../src/spaces_reader/schema'; - -import IDFetcher from '../src/id_reader/fetcher'; -import IDSlicer from '../src/id_reader/slicer'; -import IDSchema from '../src/id_reader/schema'; - -import ESStateStorageAPI from '../src/elasticsearch_state_storage/api'; -import ESStateStorageSchema from '../src/elasticsearch_state_storage/schema'; +import ESStateStorageAPI from './elasticsearch_state_storage/api.js'; +import ESStateStorageSchema from './elasticsearch_state_storage/schema.js'; // Teraslice Asset Registry // This was added to enable esbuild based bundled assets. @@ -39,11 +32,6 @@ export const ASSETS = { Processor: ESBulkSender, Schema: ESBulkSenderSchema }, - elasticsearch_data_generator: { - Fetcher: ESDataGenFetcher, - Slicer: ESDataGenSlicer, - Schema: ESDataGenSchema - }, elasticsearch_reader: { Fetcher: ESDateFetcher, Slicer: ESDateSlicer, diff --git a/asset/src/spaces_reader/fetcher.ts b/asset/src/spaces_reader/fetcher.ts index f2e9ce84a..d0f08190d 100644 --- a/asset/src/spaces_reader/fetcher.ts +++ b/asset/src/spaces_reader/fetcher.ts @@ -1,4 +1,4 @@ -import { ReaderAPIFetcher } from '../__lib/ReaderAPIFetcher'; +import { ReaderAPIFetcher } from '../__lib/ReaderAPIFetcher.js'; export default class SpacesDateFetcher extends ReaderAPIFetcher { diff --git a/asset/src/spaces_reader/schema.ts b/asset/src/spaces_reader/schema.ts index 82518fd04..bf4909f32 100644 --- a/asset/src/spaces_reader/schema.ts +++ b/asset/src/spaces_reader/schema.ts @@ -1,7 +1,7 @@ import { ConvictSchema, AnyObject, ValidatedJobConfig } from '@terascope/job-components'; -import { AssetSpacesAPIConfig } from './interfaces'; -import { opSchema } from '../__lib/schema'; -import { DEFAULT_API_NAME } from '../spaces_reader_api/interfaces'; +import { AssetSpacesAPIConfig } from './interfaces.js'; +import { opSchema } from '../__lib/schema.js'; +import { DEFAULT_API_NAME } from '../spaces_reader_api/interfaces.js'; export default class Schema extends ConvictSchema { validateJob(job: ValidatedJobConfig): void { diff --git a/asset/src/spaces_reader/slicer.ts b/asset/src/spaces_reader/slicer.ts index 0bfe56ad7..a7f8971d5 100644 --- a/asset/src/spaces_reader/slicer.ts +++ b/asset/src/spaces_reader/slicer.ts @@ -1,4 +1,4 @@ -import { DateReaderAPISlicer } from '../__lib/DateReaderAPISlicer'; +import { DateReaderAPISlicer } from '../__lib/DateReaderAPISlicer.js'; export default class SpacesDateSlicer extends DateReaderAPISlicer { diff --git a/asset/src/spaces_reader_api/schema.ts b/asset/src/spaces_reader_api/schema.ts index 71b34a680..842bb564b 100644 --- a/asset/src/spaces_reader_api/schema.ts +++ b/asset/src/spaces_reader_api/schema.ts @@ -4,8 +4,8 @@ import { } from '@terascope/job-components'; import elasticAPI from '@terascope/elasticsearch-api'; import { SpacesAPIConfig } from '@terascope/elasticsearch-asset-apis'; -import { schema } from '../elasticsearch_reader_api/schema'; -import { DEFAULT_API_NAME } from './interfaces'; +import { schema } from '../elasticsearch_reader_api/schema.js'; +import { DEFAULT_API_NAME } from './interfaces.js'; const clone = cloneDeep(schema); diff --git a/asset/tsconfig.build.json b/asset/tsconfig.build.json index ac2ed5c42..707c45543 100644 --- a/asset/tsconfig.build.json +++ b/asset/tsconfig.build.json @@ -3,9 +3,9 @@ "compilerOptions": { "rootDir": "src", "outDir": "dist", - "module": "commonjs", + "target": "ESNext", + "module": "ESNext", "moduleResolution": "node", - "target": "es2019", "skipLibCheck": true, "experimentalDecorators": true, "strict": true, diff --git a/asset/tsconfig.json b/asset/tsconfig.json index 1cac8ba0d..b9eba11bd 100644 --- a/asset/tsconfig.json +++ b/asset/tsconfig.json @@ -1,8 +1,11 @@ { "extends": "../tsconfig", "compilerOptions": { + "target": "ESNext", + "module": "ESNext", "outDir": "dist", - "rootDir": "src" + "rootDir": "src", + "esModuleInterop": true }, "include": ["src"] } diff --git a/docs/operations/elasticsearch_data_generator.md b/docs/operations/elasticsearch_data_generator.md deleted file mode 100644 index 3488f3450..000000000 --- a/docs/operations/elasticsearch_data_generator.md +++ /dev/null @@ -1,89 +0,0 @@ -# elasticsearch_data_generator # -## Deprecated, please use data_generator from standard assets -Used to generate sample data for your elasticsearch cluster. You may use the default data generator which creates randomized data fitting the format listed below or you may create your own custom schema using the [mocker-data-generator](https://github.com/danibram/mocker-data-generator) package to create data to whatever schema you desire. - -Default generated data : -```json -{ - "ip": "1.12.146.136", - "userAgent": "Mozilla/5.0 (Windows NT 5.2; WOW64; rv:8.9) Gecko/20100101 Firefox/8.9.9", - "url": "https://gabrielle.org", - "uuid": "408433ff-9495-4d1c-b066-7f9668b168f0", - "ipv6": "8188:b9ad:d02d:d69e:5ca4:05e2:9aa5:23b0", - "location": "-25.40587, 56.56418", - "created": "2016-01-19T13:33:09.356-07:00", - "bytes": 4850020 -} - -``` - -Example configuration -```json -{ - "_op": "elasticsearch_data_generator", - "size": 25000000, - "json_schema": "some/path/to/file.js", - "format": "isoBetween", - "start": "2015-08-01", - "end": "2015-12-30" -} -``` - -In once mode, this will created a total of 25 million docs with dates ranging from 2015-08-01 to 2015-12-30. The dates will appear in "2015-11-19T13:48:08.426-07:00" format. - - -| Configuration | Description | Type | Notes | -| --------- | -------- | ------ | ------ | -| _op | Name of operation, it must reflect the exact name of the file | String | required | -| size | If lifecycle is set to "once", then size is the total number of documents that the generator will make. If lifecycle is set to "persistent", then this generator will will constantly stream data to elasticsearch in chunks as big as the size indicated | Number | required | -| json_schema | File path to where custom schema is located | String | optional, the schema must be exported Node style "module.exports = schema" | -| format | specify any provided formats listed in /lib/utils/data_utils for the generator| String | optional, defaults to "dateNow" | -| start | start of date range | String | optional, only used with format isoBetween or utcBetween, defaults to Thu Jan 01 1970 00:00:00 GMT-0700 (MST) | -| end | end of date range | String | optional, only used with format isoBetween or utcBetween, defaults to new Date() | -| stress_test | If set to true, it will attempt to send non unique documents following your schema as fast as it can, originally used to help determine cluster write performance| Boolean | optional, defaults to false | -| date_key | Use this to indicate which key of your schema you would like to use a format listed below, just in case you don't want to set your own | String | optional, defaults to created | -| set_id | used to make an id on the data that will be used for the doc \_id for elasticsearch, values: base64url, hexadecimal, HEXADECIMAL | String | optional, if used, then index selector needs to have id_field set to "id" | -| id_start_key | set if you would like to force the first part of the ID to a certain character, adds a regex to the front | Sting | optional, must be used in tandem with set_id id_start_key is essentially regex, if you set it to "a", then the first character of the id will be "a", can also set ranges [a-f] or randomly alternate between b and a if its set to "[ab]" | - -#### Description of formats available #### -There are two categories of formats, ones that return the current date at which the function runs, or one that returns a date within a given range. Note for the non-range category, technically if the job takes 5 minutes to run, you will have dates ranging from the time you started the job up until the time it finished, so its still a range but not as one that spans hours, days weeks etc. - - -| Format | Description | -| --------- | -------- | -| dateNow | will create a new date in "2016-01-19T13:48:08.426-07:00" format, preserving local time | -| utcDate | will create a new utc date e.g "2016-01-19T20:48:08.426Z" | -| utcBetween | similar to utcDate, but uses start and end keys in the job config to specify range | -| isoBetween | similar to dateNow, but uses start and end keys in the job config to specify range | - - -#### persistent mode #### - The data generator will continually stream data into elasticsearch, the "size" key" switches from the total number of documents created to how big each slice is when sent to elasticsearch - - -## Example Job - -```json -{ - "name" : "testing", - "workers" : 1, - "slicers" : 1, - "lifecycle" : "once", - "assets" : [ - "elasticsearch" - ], - "operations" : [ - { - "_op": "elasticsearch_data_generator", - "size": 10000 - }, - { - "_op": "elasticsearch_bulk", - "index": "other_index", - "size": 1000, - "index": true - } - ] -} - -``` diff --git a/jest.config.js b/jest.config.js index 2b369433b..2c74225cd 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,16 +1,18 @@ -'use strict'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; -const path = require('path'); +const dirname = path.dirname(fileURLToPath(import.meta.url)); -module.exports = { +export default { verbose: true, testEnvironment: 'node', - setupFilesAfterEnv: ['jest-extended/all', '/test/test.setup.js'], + setupFilesAfterEnv: ['jest-extended/all'], collectCoverage: true, coverageReporters: ['json', 'lcov', 'text', 'html'], coverageDirectory: 'coverage', maxConcurrency: 4, workerIdleMemoryLimit: '400MB', + testTimeout: 60 * 1000, collectCoverageFrom: [ '/asset/**/*.ts', '!/packages/*/**/*.ts', @@ -25,10 +27,11 @@ module.exports = { '/test/*-spec.{ts,js}', ], moduleNameMapper: { - '^@terascope/elasticsearch-asset-apis$': path.join(__dirname, '/packages/elasticsearch-asset-apis/src/index.ts'), - '^terafoundation_elasticsearch_connector$': path.join(__dirname, '/packages/terafoundation_elasticsearch_connector/src/index.ts'), + '^@terascope/elasticsearch-asset-apis$': path.join(dirname, '/packages/elasticsearch-asset-apis/src/index.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1', }, preset: 'ts-jest', + extensionsToTreatAsEsm: ['.ts'], transform: { '\\.[jt]sx?$': ['ts-jest', { isolatedModules: true, @@ -40,6 +43,6 @@ module.exports = { }, globals: { ignoreDirectories: ['dist'], - availableExtensions: ['.js', '.ts'] + availableExtensions: ['.js', '.ts', '.mjs'] } }; diff --git a/package.json b/package.json index a7c219c3f..3480cfdf4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-assets", "displayName": "Elasticsearch Assets", - "version": "3.7.2", + "version": "4.0.0", "private": true, "description": "bundle of processors for teraslice", "homepage": "https://github.com/terascope/elasticsearch-assets#readme", @@ -14,6 +14,7 @@ }, "license": "MIT", "author": "Terascope, LLC ", + "type": "module", "workspaces": [ "packages/*", "asset" @@ -29,41 +30,38 @@ "lint:fix": "yarn lint --fix", "publish:changed": "./scripts/publish.sh", "setup": "yarn && yarn build --force", - "test": "TEST_ELASTICSEARCH='true' ts-scripts test asset --", - "test:all": "ts-scripts test", - "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug asset --", - "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ts-scripts test asset --", - "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test asset --", - "test:elasticsearch8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test asset --", - "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test asset --", - "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test asset --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test asset --", - "test:watch": "ts-scripts test --watch asset --" + "test": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test asset --", + "test:all": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test", + "test:debug": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test --debug asset --", + "test:elasticsearch6": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test asset --", + "test:elasticsearch7": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test asset --", + "test:elasticsearch8": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test asset --", + "test:opensearch1": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' ts-scripts test asset --", + "test:opensearch2": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test asset --", + "test:watch": "NODE_OPTIONS='--experimental-vm-modules' ts-scripts test --watch asset --" }, "dependencies": {}, "devDependencies": { - "@terascope/data-types": "^0.50.1", - "@terascope/elasticsearch-api": "^3.20.2", - "@terascope/elasticsearch-asset-apis": "^0.12.1", + "@terascope/data-types": "^0.50.3", + "@terascope/elasticsearch-api": "^3.20.3", + "@terascope/elasticsearch-asset-apis": "^1.0.0", "@terascope/eslint-config": "^0.8.0", - "@terascope/job-components": "^0.75.1", - "@terascope/scripts": "0.77.2", - "@terascope/teraslice-state-storage": "^0.53.2", - "@terascope/types": "^0.17.1", + "@terascope/job-components": "^1.0.1", + "@terascope/scripts": "0.77.3", + "@terascope/teraslice-state-storage": "^0.53.3", + "@terascope/types": "^0.17.3", "@types/bluebird": "^3.5.38", - "@types/elasticsearch": "^5.0.40", "@types/got": "^9.6.12", "@types/jest": "^29.5.12", - "@types/node": "^18.14.2", - "elasticsearch": "^15.4.1", - "elasticsearch-store": "^0.84.2", + "@types/node": "^20.14.6", + "elasticsearch-store": "^0.84.3", "eslint": "^8.57.0", "jest": "^29.6.4", "jest-extended": "^4.0.2", "moment": "^2.30.1", "nock": "^13.5.4", "node-notifier": "^10.0.1", - "teraslice-test-harness": "^0.30.0", + "teraslice-test-harness": "^1.0.1", "ts-jest": "^29.1.5", "typescript": "~5.2.2" }, diff --git a/packages/elasticsearch-asset-apis/jest.config.js b/packages/elasticsearch-asset-apis/jest.config.js index 20cd9fe3f..bd7c4debd 100644 --- a/packages/elasticsearch-asset-apis/jest.config.js +++ b/packages/elasticsearch-asset-apis/jest.config.js @@ -1,17 +1,18 @@ -'use strict'; - -module.exports = { +export default { verbose: true, testEnvironment: 'node', - setupFilesAfterEnv: ['jest-extended/all', '/test/test.setup.js'], + setupFilesAfterEnv: ['jest-extended/all'], collectCoverage: true, coverageReporters: ['json', 'lcov', 'text', 'html'], coverageDirectory: 'coverage', + extensionsToTreatAsEsm: ['.ts'], testMatch: [ '/test/**/*-spec.{ts,js}', '/test/*-spec.{ts,js}', ], - preset: 'ts-jest', + moduleNameMapper: { + '^(\\.{1,2}/.*)\\.js$': '$1', + }, transform: { '\\.[jt]sx?$': ['ts-jest', { isolatedModules: true, @@ -21,4 +22,5 @@ module.exports = { useESM: true }] }, + testTimeout: 60 * 1000 }; diff --git a/packages/elasticsearch-asset-apis/package.json b/packages/elasticsearch-asset-apis/package.json index 666df5ddb..a8c0496d0 100644 --- a/packages/elasticsearch-asset-apis/package.json +++ b/packages/elasticsearch-asset-apis/package.json @@ -1,13 +1,14 @@ { "name": "@terascope/elasticsearch-asset-apis", "displayName": "Elasticsearch Asset Apis", - "version": "0.12.1", + "version": "1.0.0", "description": "Elasticsearch reader and sender apis", "homepage": "https://github.com/terascope/elasticsearch-assets", "repository": "git@github.com:terascope/elasticsearch-assets.git", "license": "MIT", "author": "Terascope, LLC ", "main": "dist/src/index.js", + "type": "module", "typings": "dist/src/index.d.ts", "files": [ "dist/src/**/*" @@ -15,29 +16,25 @@ "scripts": { "build": "tsc --project tsconfig.json", "build:watch": "yarn build --watch", - "test": "TEST_ELASTICSEARCH='true' ts-scripts test . --", - "test:debug": "TEST_ELASTICSEARCH='true' ts-scripts test --debug . --", - "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ts-scripts test . --", - "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --", - "test:elasticsearch8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --", - "test:legacy": "TEST_ELASTICSEARCH='true' LEGACY_CLIENT=true ts-scripts test . --", - "test:opensearch1": "TEST_OPENSEARCH='true' ts-scripts test . --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --", - "test:watch": "ts-scripts test --watch . --" + "test": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test . --", + "test:debug": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test --debug . --", + "test:elasticsearch6": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test . --", + "test:elasticsearch7": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' ts-scripts test . --", + "test:elasticsearch8": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' ts-scripts test . --", + "test:opensearch1": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' ts-scripts test . --", + "test:opensearch2": "NODE_OPTIONS='--experimental-vm-modules' TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' ts-scripts test . --", + "test:watch": "NODE_OPTIONS='--experimental-vm-modules' TEST_ELASTICSEARCH='true' ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.56.2", - "@terascope/data-types": "^0.50.1", - "@terascope/elasticsearch-api": "^3.20.2", - "@terascope/utils": "^0.59.1", + "@terascope/data-mate": "^0.56.3", + "@terascope/data-types": "^0.50.3", + "@terascope/elasticsearch-api": "^3.20.3", + "@terascope/utils": "^0.59.3", "datemath-parser": "^1.0.6", - "got": "^11.8.3", + "got": "^13.0.0", "moment": "^2.30.1" }, - "devDependencies": { - "@types/elasticsearch": "^5.0.40", - "elasticsearch": "^15.4.1" - }, + "devDependencies": {}, "engines": { "node": ">=18.0.0", "yarn": ">=1.22.19" diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/ElasticsearchBulkSender.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/ElasticsearchBulkSender.ts index 878641d3f..78ef3b71f 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/ElasticsearchBulkSender.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/ElasticsearchBulkSender.ts @@ -5,7 +5,7 @@ import { import elasticAPI, { BulkActionMetadata, BulkRecord } from '@terascope/elasticsearch-api'; import { ElasticsearchSenderConfig, UpdateConfig -} from './interfaces'; +} from './interfaces.js'; export class ElasticsearchBulkSender implements RouteSenderAPI { client: elasticAPI.Client; diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts index 5d5796d37..ef060fe2c 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-bulk-sender/index.ts @@ -1,10 +1,10 @@ import elasticAPI from '@terascope/elasticsearch-api'; import { isNil, isPlainObject, isString } from '@terascope/utils'; -import { ElasticsearchBulkSender } from './ElasticsearchBulkSender'; -import { ElasticsearchSenderConfig } from './interfaces'; +import { ElasticsearchBulkSender } from './ElasticsearchBulkSender.js'; +import { ElasticsearchSenderConfig } from './interfaces.js'; -export * from './ElasticsearchBulkSender'; -export * from './interfaces'; +export * from './ElasticsearchBulkSender.js'; +export * from './interfaces.js'; interface BulkAPIArgs { config: ElasticsearchSenderConfig, diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts index 7b9dc4924..4060794e0 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderAPI.ts @@ -1,4 +1,4 @@ -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; import { AnyObject, DataEntity, isObjectEntity, getTypeOf, Logger, isSimpleObject, @@ -10,13 +10,13 @@ import { ClientParams, ClientResponse } from '@terascope/types'; import { DataFrame } from '@terascope/data-mate'; import { DataTypeConfig } from '@terascope/data-types'; import moment from 'moment'; -import { inspect } from 'util'; +import { inspect } from 'node:util'; import { dateSlicer, idSlicer, getKeyArray, dateOptions, processInterval, dateFormat, dateFormatSeconds, parseDate, delayedStreamSegment, determineIDSlicerRanges, determineDateSlicerRanges, -} from './algorithms'; +} from './algorithms/index.js'; import { ESReaderOptions, DateSegments, InputDateSegments, SlicerArgs, IDType, DateSlicerArgs, @@ -25,9 +25,9 @@ import { DateSlicerRange, IDSlicerRange, DateSlicerMetadata, GetIntervalResult, ReaderSlice, IDSlicerResults, FetchResponseType -} from './interfaces'; -import { WindowState } from './WindowState'; -import { buildQuery } from './utils'; +} from './interfaces.js'; +import { WindowState } from './WindowState.js'; +import { buildQuery } from './utils.js'; type FetchDate = moment.Moment | null; diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderClient.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderClient.ts index 7f27df5c9..96b4206f1 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderClient.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/ElasticsearchReaderClient.ts @@ -5,7 +5,7 @@ import { AnyObject, DataEntity, get, Logger } from '@terascope/utils'; import { DataTypeConfig, ClientParams, ClientResponse } from '@terascope/types'; -import { ReaderClient, FetchResponseType } from './interfaces'; +import { ReaderClient, FetchResponseType } from './interfaces.js'; export class ElasticsearchReaderClient implements ReaderClient { private readonly _baseClient: Client; diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts index d8ac24300..b991b0121 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/SpacesReaderClient.ts @@ -5,13 +5,15 @@ import { } from '@terascope/utils'; import { ClientParams, ClientResponse } from '@terascope/types'; import { DataTypeConfig } from '@terascope/data-types'; -import got, { OptionsOfJSONResponseBody, Response } from 'got'; +import got, { + OptionsOfJSONResponseBody, Response, TimeoutError, RequestError +} from 'got'; import { DataFrame } from '@terascope/data-mate'; -import { inspect } from 'util'; +import { inspect } from 'node:util'; import { SpacesAPIConfig, ReaderClient, FetchResponseType -} from './interfaces'; -import { throwRequestError } from './throwRequestError'; +} from './interfaces.js'; +import { throwRequestError } from './throwRequestError.js'; export class SpacesReaderClient implements ReaderClient { // NOTE: currently we are not supporting id based reader queries @@ -65,7 +67,9 @@ export class SpacesReaderClient implements ReaderClient { responseType: isJSONResponse ? 'json' : undefined, json, throwHttpErrors: false, - timeout: this.config.timeout, + timeout: { + request: this.config.timeout, + }, retry: { limit: this.retry, methods: ['POST', 'GET'], @@ -94,7 +98,7 @@ export class SpacesReaderClient implements ReaderClient { this.uri, this.getRequestOptions(query, format) ); } catch (err) { - if (err instanceof got.TimeoutError) { + if (err instanceof TimeoutError) { throw new TSError('HTTP request timed out connecting to API endpoint.', { statusCode: 408, context: { @@ -103,13 +107,15 @@ export class SpacesReaderClient implements ReaderClient { } }); } - if (err instanceof got.RequestError && err.response) { + + if (err instanceof RequestError && err.response) { throwRequestError( this.uri, err.response?.statusCode ?? 500, err.response?.body ); } + throw new TSError(err, { reason: 'Failure making search request', context: { @@ -357,7 +363,9 @@ export class SpacesReaderClient implements ReaderClient { const response = await got(uri, { searchParams: { token }, responseType: 'json', - timeout: 1000000, + timeout: { + request: 1000000 + }, retry: { limit: this.retry, methods: ['POST', 'GET'], @@ -379,7 +387,7 @@ export class SpacesReaderClient implements ReaderClient { } }; } catch (err) { - if (err instanceof got.TimeoutError) { + if (err instanceof TimeoutError) { throw new TSError('HTTP request timed out connecting to API endpoint.', { statusCode: 408, context: { diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts index 332094758..4bb115567 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/date-helpers.ts @@ -1,14 +1,14 @@ import { times, toIntegerOrThrow } from '@terascope/utils'; import moment from 'moment'; -import fs from 'fs'; +import fs from 'node:fs'; // @ts-expect-error import dateMath from 'datemath-parser'; -import { inspect } from 'util'; +import { inspect } from 'node:util'; import { StartPointConfig, SlicerDates, InputDateSegments, DateSegments, ReaderSlice, ParsedInterval, DateConfig, DateSlicerRanges, DateSlicerRange, -} from '../interfaces'; +} from '../interfaces.js'; export function dateOptions(value: string): moment.unitOfTime.Base { const options = { diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/dateSlicer.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/dateSlicer.ts index e24344c19..c3b806159 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/dateSlicer.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/dateSlicer.ts @@ -3,8 +3,8 @@ import { isString } from '@terascope/utils'; import moment from 'moment'; -import { inspect } from 'util'; -import { idSlicer } from './idSlicer'; +import { inspect } from 'node:util'; +import { idSlicer } from './idSlicer.js'; import { SlicerArgs, SlicerDateConfig, @@ -17,14 +17,14 @@ import { IDType, DateSlicerResults, ReaderSlice -} from '../interfaces'; +} from '../interfaces.js'; import { dateFormat as dFormat, dateFormatSeconds, dateOptions, determineDateSlicerRange, -} from './date-helpers'; -import { getKeyArray } from './id-helpers'; +} from './date-helpers.js'; +import { getKeyArray } from './id-helpers.js'; interface DateParams { start: moment.Moment; diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/id-helpers.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/id-helpers.ts index 664303836..89f3fc113 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/id-helpers.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/id-helpers.ts @@ -4,7 +4,7 @@ import { pMap } from '@terascope/utils'; import { CountFn, IDSlicerRanges, IDType, ReaderSlice -} from '../interfaces'; +} from '../interfaces.js'; export const base64url = Object.freeze(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/idSlicer.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/idSlicer.ts index 2a818c1ba..79a7c24e8 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/idSlicer.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/idSlicer.ts @@ -1,8 +1,8 @@ import { TSError } from '@terascope/utils'; import { IDSlicerArgs, ReaderSlice, IDSlicerResults -} from '../interfaces'; -import { generateCountQueryForKeys } from './id-helpers'; +} from '../interfaces.js'; +import { generateCountQueryForKeys } from './id-helpers.js'; export function idSlicer(args: IDSlicerArgs): () => Promise { const { diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/index.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/index.ts index 404509b65..1fafaace4 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/index.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/algorithms/index.ts @@ -1,4 +1,4 @@ -export * from './date-helpers'; -export * from './dateSlicer'; -export * from './id-helpers'; -export * from './idSlicer'; +export * from './date-helpers.js'; +export * from './dateSlicer.js'; +export * from './id-helpers.js'; +export * from './idSlicer.js'; diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/index.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/index.ts index bf5884c1b..60ad8969a 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/index.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/index.ts @@ -1,18 +1,18 @@ import { debugLogger, Logger } from '@terascope/utils'; -import { EventEmitter } from 'events'; -import { ElasticsearchReaderAPI } from './ElasticsearchReaderAPI'; +import { EventEmitter } from 'node:events'; +import { ElasticsearchReaderAPI } from './ElasticsearchReaderAPI.js'; import { ESReaderOptions, FetchResponseType, ReaderClient, SpacesAPIConfig -} from './interfaces'; -import { SpacesReaderClient } from './SpacesReaderClient'; +} from './interfaces.js'; +import { SpacesReaderClient } from './SpacesReaderClient.js'; -export * from './ElasticsearchReaderAPI'; -export * from './SpacesReaderClient'; -export * from './ElasticsearchReaderClient'; -export * from './interfaces'; -export * from './WindowState'; -export * from './algorithms'; -export * from './utils'; +export * from './ElasticsearchReaderAPI.js'; +export * from './SpacesReaderClient.js'; +export * from './ElasticsearchReaderClient.js'; +export * from './interfaces.js'; +export * from './WindowState.js'; +export * from './algorithms/index.js'; +export * from './utils.js'; export interface ElasticsearchAPIArgs { config: ESReaderOptions, diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/interfaces.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/interfaces.ts index 3060e28b8..baebc037b 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/interfaces.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/interfaces.ts @@ -1,11 +1,11 @@ import type { DataFrame } from '@terascope/data-mate'; import type { AnyObject, DataEntity, Logger } from '@terascope/utils'; -import type { EventEmitter } from 'events'; +import type { EventEmitter } from 'node:events'; import type { DataTypeConfig, xLuceneVariables, ClientParams, ClientResponse } from '@terascope/types'; -import type { WindowState } from './WindowState'; +import type { WindowState } from './WindowState.js'; /** * This is used for as the internal elasticsearch diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/throwRequestError.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/throwRequestError.ts index 8af7f3572..00ad4006a 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/throwRequestError.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/throwRequestError.ts @@ -1,7 +1,7 @@ import { has, tryParseJSON, TSError } from '@terascope/utils'; -import { inspect } from 'util'; +import { inspect } from 'node:util'; /** * Try and proxy the error as best as possible to avoid confusing errors diff --git a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts index fa3f2504f..6456b3457 100644 --- a/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts +++ b/packages/elasticsearch-asset-apis/src/elasticsearch-reader-api/utils.ts @@ -1,6 +1,6 @@ import { AnyObject, GeoPoint, ClientParams } from '@terascope/types'; import { isString, parseGeoPoint } from '@terascope/utils'; -import { ESReaderOptions, ReaderSlice } from './interfaces'; +import { ESReaderOptions, ReaderSlice } from './interfaces.js'; /** * Build the elasticsearch DSL query diff --git a/packages/elasticsearch-asset-apis/src/index.ts b/packages/elasticsearch-asset-apis/src/index.ts index 5e8063338..ee785422e 100644 --- a/packages/elasticsearch-asset-apis/src/index.ts +++ b/packages/elasticsearch-asset-apis/src/index.ts @@ -1,2 +1,2 @@ -export * from './elasticsearch-reader-api'; -export * from './elasticsearch-bulk-sender'; +export * from './elasticsearch-reader-api/index.js'; +export * from './elasticsearch-bulk-sender/index.js'; diff --git a/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts b/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts index e50829cbb..93b7bb9d9 100644 --- a/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts +++ b/packages/elasticsearch-asset-apis/test/bulk-send-spec.ts @@ -6,8 +6,8 @@ import elasticAPI from '@terascope/elasticsearch-api'; import { TEST_INDEX_PREFIX, waitForData, cleanupIndex, fetch, makeClient -} from '../test/helpers'; -import { createElasticsearchBulkSender } from '../src/elasticsearch-bulk-sender'; +} from '../test/helpers/index.js'; +import { createElasticsearchBulkSender } from '../src/elasticsearch-bulk-sender/index.js'; describe('elasticsearch bulk sender module', () => { const META_ROUTE = 'standard:route'; diff --git a/packages/elasticsearch-asset-apis/test/condensed-data-spec.ts b/packages/elasticsearch-asset-apis/test/condensed-data-spec.ts index 53e10ffb0..290bf0eb5 100644 --- a/packages/elasticsearch-asset-apis/test/condensed-data-spec.ts +++ b/packages/elasticsearch-asset-apis/test/condensed-data-spec.ts @@ -1,5 +1,5 @@ import 'jest-extended'; -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; import { debugLogger, DataEntity } from '@terascope/utils'; import { TEST_INDEX_PREFIX, @@ -8,7 +8,7 @@ import { waitForData, makeClient } from './helpers'; -import condensedData from './fixtures/data/condensed-spread'; +import condensedData from './fixtures/data/condensed-spread.js'; import { createElasticsearchReaderAPI, ESReaderOptions, @@ -18,7 +18,7 @@ import { ElasticsearchReaderClient, ReaderSlice, FetchResponseType -} from '../src'; +} from '../src/index.js'; describe('ReaderAPI with condensed time data', () => { const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_api_condensed_`; diff --git a/packages/elasticsearch-asset-apis/test/cross-cluster-spec.ts b/packages/elasticsearch-asset-apis/test/cross-cluster-spec.ts index a75011129..c8f2975d4 100644 --- a/packages/elasticsearch-asset-apis/test/cross-cluster-spec.ts +++ b/packages/elasticsearch-asset-apis/test/cross-cluster-spec.ts @@ -2,14 +2,14 @@ import 'jest-extended'; import { debugLogger, DataEntity } from '@terascope/utils'; import { DataFrame } from '@terascope/data-mate'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; import { TEST_INDEX_PREFIX, cleanupIndex, populateIndex, waitForData, makeClient -} from './helpers'; +} from './helpers/index.js'; import { createElasticsearchReaderAPI, ElasticsearchReaderClient, @@ -17,7 +17,7 @@ import { FetchResponseType, IDType, ReaderSlice -} from '../src'; +} from '../src/index.js'; describe('Reader API', () => { const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_api_dataframe_cross_cluster`; diff --git a/packages/elasticsearch-asset-apis/test/date-range/date_slicer_fn-spec.ts b/packages/elasticsearch-asset-apis/test/date-range/date_slicer_fn-spec.ts index 8c9ad67d3..565d8c611 100644 --- a/packages/elasticsearch-asset-apis/test/date-range/date_slicer_fn-spec.ts +++ b/packages/elasticsearch-asset-apis/test/date-range/date_slicer_fn-spec.ts @@ -1,5 +1,5 @@ import 'jest-extended'; -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; import { AnyObject, debugLogger, @@ -12,8 +12,8 @@ import { SlicerDateConfig, DateSegments, ReaderSlice, dateSlicer, dateFormatSeconds, dateFormat, divideRange -} from '../../src'; -import { MockClient } from '../helpers'; +} from '../../src/index.js'; +import { MockClient } from '../helpers/index.js'; interface TestConfig { slicers?: number; diff --git a/packages/elasticsearch-asset-apis/test/date-range/starting_point-spec.ts b/packages/elasticsearch-asset-apis/test/date-range/starting_point-spec.ts index d208f7285..4b27fe0ca 100644 --- a/packages/elasticsearch-asset-apis/test/date-range/starting_point-spec.ts +++ b/packages/elasticsearch-asset-apis/test/date-range/starting_point-spec.ts @@ -6,7 +6,7 @@ import { divideRange, StartPointConfig, ParsedInterval -} from '../../src'; +} from '../../src/index.js'; function makeDate(format: string) { return moment.utc(moment.utc().format(format)); diff --git a/packages/elasticsearch-asset-apis/test/fixtures/data/condensed-spread.ts b/packages/elasticsearch-asset-apis/test/fixtures/data/condensed-spread.ts index d594fc020..32eb6b832 100644 --- a/packages/elasticsearch-asset-apis/test/fixtures/data/condensed-spread.ts +++ b/packages/elasticsearch-asset-apis/test/fixtures/data/condensed-spread.ts @@ -25,8 +25,8 @@ const CondensedDataType = new DataType({ } }); -export = { +export default { data, CondensedDataType, index: 'condensed_spread' -} +}; diff --git a/packages/elasticsearch-asset-apis/test/fixtures/data/even-spread.ts b/packages/elasticsearch-asset-apis/test/fixtures/data/even-spread.ts index ddcacd991..324ebe576 100644 --- a/packages/elasticsearch-asset-apis/test/fixtures/data/even-spread.ts +++ b/packages/elasticsearch-asset-apis/test/fixtures/data/even-spread.ts @@ -10014,8 +10014,8 @@ const types: DataTypeFields = { bytes: { type: 'Integer' } }; -export = { +export default { data, types, index: 'even_spread' -} +}; diff --git a/packages/elasticsearch-asset-apis/test/helpers/elasticsearch.ts b/packages/elasticsearch-asset-apis/test/helpers/elasticsearch.ts index 2685daba0..b7b7c48d7 100644 --- a/packages/elasticsearch-asset-apis/test/helpers/elasticsearch.ts +++ b/packages/elasticsearch-asset-apis/test/helpers/elasticsearch.ts @@ -14,9 +14,6 @@ const { const logger = debugLogger('elasticsearch_helpers'); -// automatically set the timeout to 30s when using elasticsearch -jest.setTimeout(30000); - export { makeClient, formatUploadData, upload, populateIndex, waitForData, cleanupIndex, diff --git a/packages/elasticsearch-asset-apis/test/helpers/index.ts b/packages/elasticsearch-asset-apis/test/helpers/index.ts index f2528b7c3..d7e795e68 100644 --- a/packages/elasticsearch-asset-apis/test/helpers/index.ts +++ b/packages/elasticsearch-asset-apis/test/helpers/index.ts @@ -1,3 +1,3 @@ -export * from './mock-client'; -export * from './elasticsearch'; -export * from './utils'; +export * from './mock-client.js'; +export * from './elasticsearch.js'; +export * from './utils.js'; diff --git a/packages/elasticsearch-asset-apis/test/reader-api-spec.ts b/packages/elasticsearch-asset-apis/test/reader-api-spec.ts index 7e70a0b83..a8d163607 100644 --- a/packages/elasticsearch-asset-apis/test/reader-api-spec.ts +++ b/packages/elasticsearch-asset-apis/test/reader-api-spec.ts @@ -5,14 +5,14 @@ import { isElasticsearch8 } from 'elasticsearch-store'; import { DataFrame } from '@terascope/data-mate'; -import { EventEmitter } from 'events'; +import { EventEmitter } from 'node:events'; import { TEST_INDEX_PREFIX, cleanupIndex, populateIndex, waitForData, makeClient -} from './helpers'; +} from './helpers/index.js'; import { createElasticsearchReaderAPI, DateSlicerRange, @@ -22,7 +22,7 @@ import { IDType, InputDateSegments, ReaderSlice -} from '../src'; +} from '../src/index.js'; describe('Reader API', () => { const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_api_dataframe_`; diff --git a/packages/elasticsearch-asset-apis/test/spaces/spaces-client-spec.ts b/packages/elasticsearch-asset-apis/test/spaces/spaces-client-spec.ts index c79ac1528..c68e13c53 100644 --- a/packages/elasticsearch-asset-apis/test/spaces/spaces-client-spec.ts +++ b/packages/elasticsearch-asset-apis/test/spaces/spaces-client-spec.ts @@ -9,7 +9,7 @@ import { IDType, SpacesAPIConfig, SpacesReaderClient -} from '../../src'; +} from '../../src/index.js'; describe('Spaces Reader Client', () => { const baseUri = 'http://test.dev'; diff --git a/packages/elasticsearch-asset-apis/test/test.setup.js b/packages/elasticsearch-asset-apis/test/test.setup.js deleted file mode 100644 index aec2bd7fc..000000000 --- a/packages/elasticsearch-asset-apis/test/test.setup.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -process.env.NODE_ENV = 'test'; - -jest.setTimeout(15000); diff --git a/packages/elasticsearch-asset-apis/test/window_state-spec.ts b/packages/elasticsearch-asset-apis/test/window_state-spec.ts index 312ddc5c3..6ed140eaf 100644 --- a/packages/elasticsearch-asset-apis/test/window_state-spec.ts +++ b/packages/elasticsearch-asset-apis/test/window_state-spec.ts @@ -1,5 +1,5 @@ import 'jest-extended'; -import { WindowState } from '../src/elasticsearch-reader-api/WindowState'; +import { WindowState } from '../src/elasticsearch-reader-api/WindowState.js'; describe('WindowState', () => { it('can instantiate', () => { diff --git a/packages/elasticsearch-asset-apis/tsconfig.json b/packages/elasticsearch-asset-apis/tsconfig.json index d0c684985..4cffa7054 100644 --- a/packages/elasticsearch-asset-apis/tsconfig.json +++ b/packages/elasticsearch-asset-apis/tsconfig.json @@ -1,8 +1,11 @@ { "extends": "../../tsconfig", "compilerOptions": { + "target": "ESNext", + "module": "ESNext", "outDir": "dist", - "rootDir": "." + "rootDir": ".", + "esModuleInterop": true }, "include": ["src", "test"] } diff --git a/packages/terafoundation_elasticsearch_connector/jest.config.js b/packages/terafoundation_elasticsearch_connector/jest.config.js deleted file mode 100644 index a6dd649ca..000000000 --- a/packages/terafoundation_elasticsearch_connector/jest.config.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict'; - -module.exports = { - verbose: true, - testEnvironment: 'node', - setupFilesAfterEnv: ['jest-extended/all'], - collectCoverage: true, - coverageReporters: ['json', 'lcov', 'text', 'html'], - coverageDirectory: 'coverage', - testMatch: [ - '/test/**/*-spec.{ts,js}', - '/test/*-spec.{ts,js}', - ], - preset: 'ts-jest', - globals: { - 'ts-jest': { - tsconfig: './tsconfig.json', - diagnostics: true, - pretty: true, - } - } -}; diff --git a/packages/terafoundation_elasticsearch_connector/package.json b/packages/terafoundation_elasticsearch_connector/package.json deleted file mode 100644 index 7f210bfe2..000000000 --- a/packages/terafoundation_elasticsearch_connector/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "terafoundation_elasticsearch_connector", - "displayName": "Terafoundation Elasticsearch Connector", - "version": "0.1.6", - "description": "Elasticsearch connector for terafoundation", - "homepage": "https://github.com/terascope/terafoundation_elasticsearch_connector", - "repository": "git@github.com:terascope/terafoundation_elasticsearch_connector.git", - "license": "MIT", - "author": "Terascope, LLC ", - "main": "dist/src/index.js", - "typings": "dist/src/index.d.ts", - "files": [ - "dist/src/**/*" - ], - "scripts": { - "build": "tsc --project tsconfig.json", - "build:watch": "yarn build --watch", - "test": "jest", - "test:debug": "env DEBUG='*teraslice*' jest --detectOpenHandles --coverage=false --runInBand", - "test:watch": "jest --coverage=false --notify --watch --onlyChanged" - }, - "dependencies": { - "@elastic/elasticsearch": "7.13.0", - "@opensearch-project/opensearch": "^1.1.0", - "@terascope/utils": "^0.59.1", - "elasticsearch": "^15.4.1" - }, - "devDependencies": { - "@types/elasticsearch": "^5.0.40" - }, - "engines": { - "node": ">=18.0.0", - "yarn": ">=1.22.19" - }, - "publishConfig": { - "access": "public", - "registry": "https://registry.npmjs.org/" - }, - "srcMain": "src/index.ts", - "terascope": { - "testSuite": "search" - } -} diff --git a/packages/terafoundation_elasticsearch_connector/src/index.ts b/packages/terafoundation_elasticsearch_connector/src/index.ts deleted file mode 100644 index 869b18405..000000000 --- a/packages/terafoundation_elasticsearch_connector/src/index.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Logger, pDefer } from '@terascope/utils'; -import type { Client, ConfigOptions } from 'elasticsearch'; -import schema from './schema'; - -function logWrapper(logger: Logger) { - return function _logger() { - return { - error: logger.error.bind(logger), - warning: logger.warn.bind(logger), - info: logger.info.bind(logger), - debug: logger.debug.bind(logger), - trace( - method: any, - requestUrl: any, - body: any, - responseBody: any, - responseStatus: any - ) { - logger.trace({ - method, - requestUrl, - body, - responseBody, - responseStatus - }); - }, - close() {} - }; - }; -} - -function create(customConfig: ConfigOptions, logger: Logger): { - client: Client, - log: any; -} { - const elasticsearch = require('elasticsearch'); - - logger.info(`using elasticsearch hosts: ${customConfig.host}`); - - customConfig.defer = pDefer; - - const client = new elasticsearch.Client(customConfig); - - return { - client, - log: logWrapper(logger) - }; -} - -export default { - create, - config_schema(): Record { - return schema; - } -}; diff --git a/packages/terafoundation_elasticsearch_connector/src/schema.ts b/packages/terafoundation_elasticsearch_connector/src/schema.ts deleted file mode 100644 index 020e58f57..000000000 --- a/packages/terafoundation_elasticsearch_connector/src/schema.ts +++ /dev/null @@ -1,32 +0,0 @@ -export = { - host: { - doc: 'A list of hosts to connect to', - default: ['127.0.0.1:9200'] - }, - sniffOnStart: { - doc: 'Sniff hosts on start up', - default: false - }, - sniffOnConnectionFault: { - doc: 'Sniff hosts on connection failure', - default: false - }, - apiVersion: { - describe: 'The API version, currently we only support 5.6, 6.5 and 7.0', - default: '6.5' - }, - requestTimeout: { - doc: 'Request timeout', - default: 120000, - format: 'duration' - }, - deadTimeout: { - doc: 'Timeout before marking a connection as "dead"', - default: 30000, - format: 'duration' - }, - maxRetries: { - doc: 'Maximum retries for a failed request', - default: 3 - } -}; diff --git a/packages/terafoundation_elasticsearch_connector/test/connector-spec.ts b/packages/terafoundation_elasticsearch_connector/test/connector-spec.ts deleted file mode 100644 index 248247ec6..000000000 --- a/packages/terafoundation_elasticsearch_connector/test/connector-spec.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { debugLogger } from '@terascope/utils'; -import 'jest-extended'; -import connector from '../src'; - -describe('terafoundation_elasticsearch_connector', () => { - const logger = debugLogger('connector-test'); - - it('has create and config_schema methods', () => { - expect(connector.config_schema).toBeFunction(); - expect(connector.config_schema).toBeFunction(); - }); - - it('config_schema returns a schema', () => { - const schema = connector.config_schema(); - - expect(schema).toBeObject(); - }); - - it('create returns a client and a logger', () => { - const { client, log } = connector.create({}, logger); - - expect(client).toBeDefined(); - expect(log).toBeDefined(); - }); -}); diff --git a/packages/terafoundation_elasticsearch_connector/tsconfig.json b/packages/terafoundation_elasticsearch_connector/tsconfig.json deleted file mode 100644 index d0c684985..000000000 --- a/packages/terafoundation_elasticsearch_connector/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../../tsconfig", - "compilerOptions": { - "outDir": "dist", - "rootDir": "." - }, - "include": ["src", "test"] -} diff --git a/test/elasticsearch_bulk/processor-spec.ts b/test/elasticsearch_bulk/processor-spec.ts index 3e09950b3..98ef3bb39 100644 --- a/test/elasticsearch_bulk/processor-spec.ts +++ b/test/elasticsearch_bulk/processor-spec.ts @@ -1,12 +1,14 @@ import 'jest-extended'; - import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; import { ClientParams } from '@terascope/types'; -import { DataEntity, OpConfig } from '@terascope/job-components'; +import { + DataEntity, OpConfig, debugLogger, + TestClientConfig +} from '@terascope/job-components'; import { makeClient, cleanupIndex, fetch, upload, waitForData, TEST_INDEX_PREFIX, -} from '../helpers'; +} from '../helpers/index.js'; interface ClientCalls { [key: string]: ClientParams.BulkParams @@ -15,9 +17,10 @@ interface ClientCalls { describe('elasticsearch_bulk', () => { const bulkIndex = `${TEST_INDEX_PREFIX}_bulk_`; const docType = '_doc'; + const logger = debugLogger('test-logger'); let harness: WorkerTestHarness; - let clients: any; + let clients: TestClientConfig[]; let clientCalls: ClientCalls = {}; let esClient: any; @@ -59,14 +62,16 @@ describe('elasticsearch_bulk', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: defaultClient + client: defaultClient, + logger }), }, { type: 'elasticsearch-next', endpoint: 'otherConnection', createClient: async () => ({ - client: otherClient + client: otherClient, + logger }), } ]; diff --git a/test/elasticsearch_bulk/schema-spec.ts b/test/elasticsearch_bulk/schema-spec.ts index 85fb3defb..1cc5c1819 100644 --- a/test/elasticsearch_bulk/schema-spec.ts +++ b/test/elasticsearch_bulk/schema-spec.ts @@ -1,40 +1,44 @@ import 'jest-extended'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { AnyObject } from '@terascope/job-components'; -import { ElasticsearchBulkConfig } from '../../asset/src/elasticsearch_bulk/interfaces'; -import { DEFAULT_API_NAME } from '../../asset/src/elasticsearch_sender_api/interfaces'; +import { debugLogger, OpConfig, TestClientConfig } from '@terascope/job-components'; +import { ElasticsearchBulkConfig } from '../../asset/src/elasticsearch_bulk/interfaces.js'; +import { DEFAULT_API_NAME } from '../../asset/src/elasticsearch_sender_api/interfaces.js'; describe('Elasticsearch Bulk Schema', () => { const index = 'some_index'; const name = 'elasticsearch_bulk'; + const logger = debugLogger('test-logger'); - const clients = [ + const clients: TestClientConfig[] = [ { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: {} + client: {}, + logger }), }, ]; let harness: WorkerTestHarness; - async function makeSchema(config: AnyObject = {}): Promise { + async function makeSchema(config: Record = {}): Promise { const opConfig = Object.assign({}, { _op: name, index }, config); harness = WorkerTestHarness.testProcessor(opConfig, { clients }); await harness.initialize(); const validConfig = harness.executionContext.config.operations.find( - (testConfig) => testConfig._op === name + (testConfig: OpConfig) => testConfig._op === name ); return validConfig as ElasticsearchBulkConfig; } afterEach(async () => { - if (harness) await harness.shutdown(); + if (harness) { + await harness.shutdown(); + } }); describe('when validating the schema', () => { diff --git a/test/elasticsearch_data_generator/fetcher-spec.ts b/test/elasticsearch_data_generator/fetcher-spec.ts deleted file mode 100644 index 6dad793d7..000000000 --- a/test/elasticsearch_data_generator/fetcher-spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { WorkerTestHarness } from 'teraslice-test-harness'; - -describe('elasticsearch_data_generator fetcher', () => { - let harness: WorkerTestHarness; - let clients: any; - - afterEach(async () => { - if (harness) await harness.shutdown(); - }); - - async function makeFetcherTest(config: any) { - harness = WorkerTestHarness.testFetcher(config, { clients }); - await harness.initialize(); - return harness; - } - - it('returns a function that produces generated data', async () => { - const opConfig = { _op: 'elasticsearch_data_generator' }; - - const test = await makeFetcherTest(opConfig); - const results = await test.runSlice({ count: 1 }); - - expect(results.length).toEqual(1); - expect(Object.keys(results[0]).length).toBeGreaterThan(1); - }); -}); diff --git a/test/elasticsearch_data_generator/slicer-spec.ts b/test/elasticsearch_data_generator/slicer-spec.ts deleted file mode 100644 index 86e7585ba..000000000 --- a/test/elasticsearch_data_generator/slicer-spec.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ - -import { SlicerRecoveryData, LifeCycle } from '@terascope/job-components'; -import { SlicerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; - -interface SlicerTestArgs { - opConfig: any; - numOfSlicers?: number; - recoveryData?: SlicerRecoveryData[]; - lifecycle?: LifeCycle; - size: number; -} - -describe('elasticsearch_data_generator', () => { - let harness: SlicerTestHarness; - let clients: any; - - afterEach(async () => { - if (harness) await harness.shutdown(); - }); - - async function makeSlicerTest({ - opConfig, - numOfSlicers = 1, - recoveryData, - lifecycle = 'once', - size - }: SlicerTestArgs) { - const job = newTestJobConfig({ - analytics: true, - slicers: numOfSlicers, - lifecycle, - operations: [ - opConfig, - { - _op: 'noop', - size - } - ], - }); - harness = new SlicerTestHarness(job, { clients }); - await harness.initialize(recoveryData); - return harness; - } - - describe('slicer', () => { - it('in "once" mode will return number based off total size of last op', async () => { - const opConfig = { _op: 'elasticsearch_data_generator', size: 15 }; - const config: SlicerTestArgs = { - opConfig, - numOfSlicers: 1, - size: 5 - }; - - const test = await makeSlicerTest(config); - - const [slice1] = await test.createSlices(); - expect(slice1).toEqual({ count: 5, processed: 5 }); - - const [slice2] = await test.createSlices(); - expect(slice2).toEqual({ count: 5, processed: 10 }); - - const [slice3] = await test.createSlices(); - expect(slice3).toEqual({ count: 5, processed: 15 }); - - const [slice4] = await test.createSlices(); - expect(slice4).toEqual(null); - }); - - it('in "once" mode can deal with uneven slice numbers', async () => { - const opConfig = { _op: 'elasticsearch_data_generator', size: 12 }; - const config: SlicerTestArgs = { - opConfig, - numOfSlicers: 1, - size: 5 - }; - const test = await makeSlicerTest(config); - - const [slice1] = await test.createSlices(); - expect(slice1).toEqual({ count: 5, processed: 5 }); - - const [slice2] = await test.createSlices(); - expect(slice2).toEqual({ count: 5, processed: 10 }); - - const [slice3] = await test.createSlices(); - expect(slice3).toEqual({ count: 2, processed: 12 }); - - const [slice4] = await test.createSlices(); - expect(slice4).toEqual(null); - }); - - it('can run recovery in once mode', async () => { - const opConfig = { _op: 'elasticsearch_data_generator', size: 15 }; - const config: SlicerTestArgs = { - opConfig, - numOfSlicers: 1, - recoveryData: [{ lastSlice: { count: 5, processed: 5 }, slicer_id: 0 }], - size: 5 - }; - const test = await makeSlicerTest(config); - - const [slice1] = await test.createSlices(); - expect(slice1).toEqual({ count: 5, processed: 10 }); - - const [slice2] = await test.createSlices(); - expect(slice2).toEqual({ count: 5, processed: 15 }); - - const [slice3] = await test.createSlices(); - expect(slice3).toEqual(null); - }); - - it('in "once" mode will return number based off total size, which can consume it all', async () => { - const opConfig = { _op: 'elasticsearch_data_generator', size: 15 }; - const config: SlicerTestArgs = { - opConfig, - numOfSlicers: 1, - size: 5000 - }; - const test = await makeSlicerTest(config); - - const [slice1] = await test.createSlices(); - expect(slice1).toEqual({ count: 15, processed: 15 }); - - const [slice2] = await test.createSlices(); - expect(slice2).toEqual(null); - - const [slice3] = await test.createSlices(); - expect(slice3).toEqual(null); - - const [slice4] = await test.createSlices(); - expect(slice4).toEqual(null); - }); - - it('slicer in "persistent" mode will continuously produce the same number', async () => { - const opConfig = { _op: 'elasticsearch_data_generator', size: 550 }; - const config: SlicerTestArgs = { - lifecycle: 'persistent', - numOfSlicers: 1, - opConfig, - size: 5000 - }; - const testHarness = await makeSlicerTest(config); - - const results1 = await testHarness.createSlices(); - const results2 = await testHarness.createSlices(); - const results3 = await testHarness.createSlices(); - - expect(results1).toEqual([{ count: 5000, processed: 5000 }]); - expect(results2).toEqual([{ count: 5000, processed: 10000 }]); - expect(results3).toEqual([{ count: 5000, processed: 15000 }]); - }); - - it('data generator will only return one slicer', async () => { - const opConfig = { _op: 'elasticsearch_data_generator', size: 550 }; - const config: SlicerTestArgs = { - lifecycle: 'persistent', - numOfSlicers: 3, - opConfig, - size: 5000 - }; - const testHarness = await makeSlicerTest(config); - const slicer = testHarness.slicer(); - - expect(slicer.slicers()).toEqual(1); - }); - }); -}); diff --git a/test/elasticsearch_reader/fetcher-spec.ts b/test/elasticsearch_reader/fetcher-spec.ts index b6fbd6bae..dea1ef1d8 100644 --- a/test/elasticsearch_reader/fetcher-spec.ts +++ b/test/elasticsearch_reader/fetcher-spec.ts @@ -1,15 +1,16 @@ import 'jest-extended'; -import { DataEntity, AnyObject } from '@terascope/job-components'; +import { DataEntity, TestClientConfig, debugLogger } from '@terascope/job-components'; import { WorkerTestHarness, newTestJobConfig, JobTestHarness } from 'teraslice-test-harness'; import { ElasticsearchTestHelpers, isOpensearch2, isElasticsearch8 } from 'elasticsearch-store'; import { TEST_INDEX_PREFIX, makeClient, cleanupIndex, populateIndex, addToIndex -} from '../helpers'; -import evenSpreadExtra1 from '../fixtures/data/even-spread-extra1'; +} from '../helpers/index.js'; +import evenSpreadExtra1 from '../fixtures/data/even-spread-extra1.js'; describe('elasticsearch_reader fetcher', () => { const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_fetcher_`; + const logger = debugLogger('test-logger'); function makeIndex(str: string) { return `${readerIndex}_${str}`; @@ -23,7 +24,7 @@ describe('elasticsearch_reader fetcher', () => { let workerHarness: WorkerTestHarness; let jobHarness: JobTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; let hasTypedDoc = true; beforeAll(async () => { @@ -38,7 +39,8 @@ describe('elasticsearch_reader fetcher', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }), }, @@ -69,7 +71,7 @@ describe('elasticsearch_reader fetcher', () => { type: docType }; - async function makeFetcherTest(config: AnyObject = {}) { + async function makeFetcherTest(config: Record = {}) { const opConfig = Object.assign({}, defaults, config); workerHarness = WorkerTestHarness.testFetcher(opConfig, { clients }); @@ -78,7 +80,7 @@ describe('elasticsearch_reader fetcher', () => { return workerHarness; } - async function makeJobTest(config: AnyObject = {}) { + async function makeJobTest(config: Record = {}) { const opConfig = Object.assign({}, defaults, config); const job = newTestJobConfig({ operations: [ diff --git a/test/elasticsearch_reader/job-spec.ts b/test/elasticsearch_reader/job-spec.ts index bf6e2bfc3..68589c67a 100644 --- a/test/elasticsearch_reader/job-spec.ts +++ b/test/elasticsearch_reader/job-spec.ts @@ -1,13 +1,14 @@ import 'jest-extended'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; import { JobTestHarness, newTestJobConfig } from 'teraslice-test-harness'; +import { debugLogger, TestClientConfig } from '@terascope/job-components'; import { TEST_INDEX_PREFIX, getTotalSliceCounts, makeClient, cleanupIndex, populateIndex -} from '../helpers'; +} from '../helpers/index.js'; describe('date_reader job', () => { const idIndex = `${TEST_INDEX_PREFIX}_es_reader_job_`; @@ -15,6 +16,7 @@ describe('date_reader job', () => { const evenSpread = ElasticsearchTestHelpers.EvenDateData; const docType = '_doc'; const bulkData = evenSpread.data; + const logger = debugLogger('test-logger'); function makeIndex(str: string) { return `${idIndex}_${str}`; @@ -23,7 +25,7 @@ describe('date_reader job', () => { const evenIndex = makeIndex('even_spread-2020.0.1'); let harness: JobTestHarness; - let clients: any; + let clients: TestClientConfig[]; let esClient: any; beforeAll(async () => { @@ -42,14 +44,16 @@ describe('date_reader job', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) }, { type: 'elasticsearch-next', endpoint: 'otherConnection', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; diff --git a/test/elasticsearch_reader/schema-spec.ts b/test/elasticsearch_reader/schema-spec.ts index e44e3af8d..033dd1046 100644 --- a/test/elasticsearch_reader/schema-spec.ts +++ b/test/elasticsearch_reader/schema-spec.ts @@ -1,20 +1,24 @@ import 'jest-extended'; -import { AnyObject, newTestJobConfig } from '@terascope/job-components'; +import { + newTestJobConfig, OpConfig, APIConfig, + debugLogger, TestClientConfig +} from '@terascope/job-components'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; import { WorkerTestHarness } from 'teraslice-test-harness'; -import { ESReaderConfig } from '../../asset/src/elasticsearch_reader/interfaces'; -import * as ESReaderSchema from '../../asset/src/elasticsearch_reader_api/schema'; -import { DEFAULT_API_NAME } from '../../asset/src/elasticsearch_reader_api/interfaces'; +import { ESReaderConfig } from '../../asset/src/elasticsearch_reader/interfaces.js'; +import * as ESReaderSchema from '../../asset/src/elasticsearch_reader_api/schema.js'; +import { DEFAULT_API_NAME } from '../../asset/src/elasticsearch_reader_api/interfaces.js'; import { TEST_INDEX_PREFIX, makeClient, cleanupIndex, populateIndex -} from '../helpers'; +} from '../helpers/index.js'; describe('elasticsearch_reader schema', () => { const name = 'elasticsearch_reader'; const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_reader_schema`; + const logger = debugLogger('test-logger'); function makeIndex(str: string) { return `${readerIndex}_${str}`; @@ -25,7 +29,7 @@ describe('elasticsearch_reader schema', () => { const index = makeIndex('even_spread'); const evenBulkData = evenSpread.data; - let clients: any; + let clients: TestClientConfig[]; let harness: WorkerTestHarness; let esClient: any; @@ -36,7 +40,8 @@ describe('elasticsearch_reader schema', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; @@ -56,7 +61,7 @@ describe('elasticsearch_reader schema', () => { } }); - async function makeSchema(config: AnyObject = {}): Promise { + async function makeSchema(config: Record = {}): Promise { const opConfig = Object.assign({}, { _op: name, index, @@ -68,7 +73,7 @@ describe('elasticsearch_reader schema', () => { await harness.initialize(); const validConfig = harness.executionContext.config.operations.find( - (testConfig) => testConfig._op === name + (testConfig: OpConfig) => testConfig._op === name ); return validConfig as ESReaderConfig; @@ -193,7 +198,10 @@ describe('elasticsearch_reader schema', () => { ] }); - expect(() => new WorkerTestHarness(job, { clients })).toThrow(); + await expect(async () => { + const test = new WorkerTestHarness(job, { clients }); + await test.initialize(); + }).rejects.toThrow(); }); it('should not throw if base api is created but opConfig has index set to another value', async () => { @@ -222,7 +230,7 @@ describe('elasticsearch_reader schema', () => { await harness.initialize(); const apiConfig = harness.executionContext.config.apis.find( - (api) => api._name === 'elasticsearch_reader_api:elasticsearch_reader-0' + (api: APIConfig) => api._name === 'elasticsearch_reader_api:elasticsearch_reader-0' ); expect(apiConfig).toMatchObject({ index }); diff --git a/test/elasticsearch_reader/slicer-spec.ts b/test/elasticsearch_reader/slicer-spec.ts index 5dfaf6cc5..ae360e852 100644 --- a/test/elasticsearch_reader/slicer-spec.ts +++ b/test/elasticsearch_reader/slicer-spec.ts @@ -1,7 +1,8 @@ import 'jest-extended'; import { pDelay, LifeCycle, SlicerRecoveryData, - AnyObject, sortBy, SliceRequest + AnyObject, sortBy, SliceRequest, debugLogger, + TestClientConfig } from '@terascope/job-components'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; import moment from 'moment'; @@ -12,10 +13,11 @@ import { makeClient, cleanupIndex, populateIndex, -} from '../helpers'; +} from '../helpers/index.js'; describe('elasticsearch_reader slicer', () => { const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_slicer_`; + const logger = debugLogger('test-logger'); function makeIndex(str: string) { return `${readerIndex}_${str}`; @@ -53,7 +55,7 @@ describe('elasticsearch_reader slicer', () => { let harness: SlicerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -63,7 +65,8 @@ describe('elasticsearch_reader slicer', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; diff --git a/test/elasticsearch_reader_api/api-spec.ts b/test/elasticsearch_reader_api/api-spec.ts index 82b823ac6..829ac3eeb 100644 --- a/test/elasticsearch_reader_api/api-spec.ts +++ b/test/elasticsearch_reader_api/api-spec.ts @@ -1,23 +1,24 @@ import 'jest-extended'; -import { DataEntity } from '@terascope/job-components'; +import { DataEntity, debugLogger, TestClientConfig } from '@terascope/job-components'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { ElasticReaderFactoryAPI } from '../../asset/src/elasticsearch_reader_api/interfaces'; +import { ElasticReaderFactoryAPI } from '../../asset/src/elasticsearch_reader_api/interfaces.js'; import { TEST_INDEX_PREFIX, cleanupIndex, makeClient, upload, waitForData -} from '../helpers'; +} from '../helpers/index.js'; describe('elasticsearch reader api', () => { const apiReaderIndex = `${TEST_INDEX_PREFIX}_reader_api_`; + const logger = debugLogger('test-logger'); const docType = '_doc'; let harness: WorkerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -27,7 +28,8 @@ describe('elasticsearch reader api', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }), } ]; diff --git a/test/elasticsearch_reader_api/schema-spec.ts b/test/elasticsearch_reader_api/schema-spec.ts index 855c3dc37..d5a458bec 100644 --- a/test/elasticsearch_reader_api/schema-spec.ts +++ b/test/elasticsearch_reader_api/schema-spec.ts @@ -1,18 +1,19 @@ import 'jest-extended'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { AnyObject } from '@terascope/job-components'; -import { TEST_INDEX_PREFIX, makeClient } from '../helpers'; -import { ElasticsearchReaderAPIConfig, DEFAULT_API_NAME } from '../../asset/src/elasticsearch_reader_api/interfaces'; +import { TestClientConfig, APIConfig, debugLogger } from '@terascope/job-components'; +import { TEST_INDEX_PREFIX, makeClient } from '../helpers/index.js'; +import { ElasticsearchReaderAPIConfig, DEFAULT_API_NAME } from '../../asset/src/elasticsearch_reader_api/interfaces.js'; describe('elasticsearch reader api schema', () => { const apiSenderIndex = `${TEST_INDEX_PREFIX}_elasticsearch_reader_api_schema_`; const docType = '_doc'; + const logger = debugLogger('test-logger'); const apiName = DEFAULT_API_NAME; const harnesses: WorkerTestHarness[] = []; let harness: WorkerTestHarness; - let clients: any; + let clients: TestClientConfig[]; let esClient: any; beforeAll(async () => { @@ -22,13 +23,16 @@ describe('elasticsearch reader api schema', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; }); - async function makeSchema(config: AnyObject = {}): Promise { + async function makeSchema( + config: Record = {} + ): Promise { const defaults = { _name: apiName, type: docType, @@ -56,7 +60,10 @@ describe('elasticsearch reader api schema', () => { harnesses.push(harness); const { apis } = harness.executionContext.config; - return apis.find((settings) => settings._name === apiName) as ElasticsearchReaderAPIConfig; + + return apis.find( + (settings: APIConfig) => settings._name === apiName + ) as ElasticsearchReaderAPIConfig; } afterEach(async () => { diff --git a/test/elasticsearch_sender_api/api-spec.ts b/test/elasticsearch_sender_api/api-spec.ts index d76fefc43..53c895236 100644 --- a/test/elasticsearch_sender_api/api-spec.ts +++ b/test/elasticsearch_sender_api/api-spec.ts @@ -1,17 +1,19 @@ import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { isNil } from '@terascope/job-components'; +import { isNil, debugLogger, TestClientConfig } from '@terascope/job-components'; import { - TEST_INDEX_PREFIX, cleanupIndex, makeClient, fetch, waitForData -} from '../helpers'; -import { ElasticSenderAPI } from '../../asset/src/elasticsearch_sender_api/interfaces'; + TEST_INDEX_PREFIX, cleanupIndex, makeClient, + fetch, waitForData +} from '../helpers/index.js'; +import { ElasticSenderAPI } from '../../asset/src/elasticsearch_sender_api/interfaces.js'; describe('elasticsearch sender api', () => { const apiSendIndex = `${TEST_INDEX_PREFIX}_send_api_`; const docType = '_doc'; + const logger = debugLogger('test-logger'); let harness: WorkerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -20,7 +22,9 @@ describe('elasticsearch sender api', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger + }), } ]; @@ -59,6 +63,8 @@ describe('elasticsearch sender api', () => { harness = new WorkerTestHarness(job, { clients }); + await harness.initialize(); + const processor = harness.getOperation('noop'); // @ts-expect-error\ processor.onBatch = async function test(data: DataEntity[]) { @@ -71,8 +77,6 @@ describe('elasticsearch sender api', () => { return data; }; - await harness.initialize(); - return harness; } diff --git a/test/elasticsearch_sender_api/schema-spec.ts b/test/elasticsearch_sender_api/schema-spec.ts index c3f672e3f..ce650de13 100644 --- a/test/elasticsearch_sender_api/schema-spec.ts +++ b/test/elasticsearch_sender_api/schema-spec.ts @@ -1,18 +1,25 @@ import 'jest-extended'; -import path from 'path'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { AnyObject, TestContext } from '@terascope/job-components'; -import { TEST_INDEX_PREFIX, makeClient } from '../helpers'; -import { ElasticsearchSenderAPI, DEFAULT_API_NAME } from '../../asset/src/elasticsearch_sender_api/interfaces'; -import SenderSchema from '../../asset/src/elasticsearch_sender_api/schema'; +import { + TestContext, APIConfig, Context, + debugLogger, TestClientConfig +} from '@terascope/job-components'; +import { TEST_INDEX_PREFIX, makeClient } from '../helpers/index.js'; +import { ElasticsearchSenderAPI, DEFAULT_API_NAME } from '../../asset/src/elasticsearch_sender_api/interfaces.js'; +import SenderSchema from '../../asset/src/elasticsearch_sender_api/schema.js'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); describe('elasticsearch sender api schema', () => { const apiSenderIndex = `${TEST_INDEX_PREFIX}_elasticsearch_sender_api_schema_`; const docType = '_doc'; + const logger = debugLogger('test-logger'); let harness: WorkerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -22,7 +29,8 @@ describe('elasticsearch sender api schema', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }), } ]; @@ -30,7 +38,7 @@ describe('elasticsearch sender api schema', () => { const apiName = DEFAULT_API_NAME; - async function makeSchema(config: AnyObject = {}): Promise { + async function makeSchema(config: Record = {}): Promise { const defaults = { _name: apiName, type: docType, @@ -58,11 +66,15 @@ describe('elasticsearch sender api schema', () => { await harness.initialize(); const { apis } = harness.executionContext.config; - return apis.find((settings) => settings._name === apiName) as ElasticsearchSenderAPI; + return apis.find( + (settings: APIConfig) => settings._name === apiName + ) as ElasticsearchSenderAPI; } afterEach(async () => { - await harness.shutdown(); + if (harness) { + await harness.shutdown(); + } }); it('should have defaults', async () => { @@ -99,29 +111,32 @@ describe('elasticsearch sender api schema', () => { describe('elasticsearch sender api schema for routed sender jobs', () => { let harness: WorkerTestHarness; let context: TestContext; + const logger = debugLogger('test-logger2'); beforeAll(async () => { // need this to create valid context to remove the default es connection const esClient = await makeClient(); - const clients = [ + const clients: TestClientConfig[] = [ { type: 'elasticsearch-next', endpoint: 'test-es', createClient: async () => ({ - client: esClient + client: esClient, + logger }), }, { type: 'elasticsearch-next', endpoint: 'test-es1', createClient: async () => ({ - client: esClient + client: esClient, + logger }), } ]; - const testAsset = path.join(__dirname, '..', 'fixtures'); + const testAsset = path.join(dirname, '..', 'fixtures'); const initialJob = newTestJobConfig({ max_retries: 3, @@ -162,7 +177,7 @@ describe('elasticsearch sender api schema for routed sender jobs', () => { }); it('should not throw if default is not an es endpoint and routed sender is an operation', async () => { - const schema = new SenderSchema(context); + const schema = new SenderSchema(context as unknown as Context); const job = newTestJobConfig({ max_retries: 3, @@ -192,7 +207,7 @@ describe('elasticsearch sender api schema for routed sender jobs', () => { }); it('should not throw if default is not an es endpoint and multiple routed sender operations', async () => { - const schema = new SenderSchema(context); + const schema = new SenderSchema(context as unknown as Context); const job = newTestJobConfig({ max_retries: 3, @@ -266,7 +281,7 @@ describe('elasticsearch sender api schema for routed sender jobs', () => { ], }); - const schema = new SenderSchema(context); + const schema = new SenderSchema(context as unknown as Context); expect(() => schema.validateJob(job2)).toThrow(); }); diff --git a/test/elasticsearch_state_storage_api/api-spec.ts b/test/elasticsearch_state_storage_api/api-spec.ts index 902753a98..64a96b171 100644 --- a/test/elasticsearch_state_storage_api/api-spec.ts +++ b/test/elasticsearch_state_storage_api/api-spec.ts @@ -1,6 +1,6 @@ import 'jest-extended'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { DataEntity } from '@terascope/job-components'; +import { DataEntity, TestClientConfig, debugLogger } from '@terascope/job-components'; import { ESCachedStateStorage } from '@terascope/teraslice-state-storage'; import { TEST_INDEX_PREFIX, @@ -8,12 +8,13 @@ import { makeClient, waitForData, fetch -} from '../helpers'; +} from '../helpers/index.js'; describe('elasticsearch state storage api', () => { const idField = '_key'; const apiReaderIndex = `${TEST_INDEX_PREFIX}_state__storage_api_`; const docType = '_doc'; + const logger = debugLogger('test-logger'); let api: ESCachedStateStorage; @@ -61,7 +62,7 @@ describe('elasticsearch state storage api', () => { let harness: WorkerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -71,7 +72,8 @@ describe('elasticsearch state storage api', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }), } ]; diff --git a/test/elasticsearch_state_storage_api/schema-spec.ts b/test/elasticsearch_state_storage_api/schema-spec.ts index 04b9f773e..22bc4849b 100644 --- a/test/elasticsearch_state_storage_api/schema-spec.ts +++ b/test/elasticsearch_state_storage_api/schema-spec.ts @@ -1,17 +1,18 @@ import 'jest-extended'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { AnyObject } from '@terascope/job-components'; -import { TEST_INDEX_PREFIX, makeClient } from '../helpers'; -import { ESStateStorageConfig } from '../../asset/src/elasticsearch_state_storage/interfaces'; +import { APIConfig, debugLogger, TestClientConfig } from '@terascope/job-components'; +import { TEST_INDEX_PREFIX, makeClient } from '../helpers/index.js'; +import { ESStateStorageConfig } from '../../asset/src/elasticsearch_state_storage/interfaces.js'; describe('elasticsearch state storage api schema', () => { const apiReaderIndex = `${TEST_INDEX_PREFIX}_state__storage_api_`; const docType = '_doc'; const apiName = 'elasticsearch_state_storage'; + const logger = debugLogger('test-logger'); let harness: WorkerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -21,14 +22,15 @@ describe('elasticsearch state storage api schema', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }), } ]; }); - async function makeSchema(config: AnyObject = {}): Promise { - const base: AnyObject = { + async function makeSchema(config: Record = {}): Promise { + const base: Record = { _name: apiName, index: apiReaderIndex, type: docType, @@ -56,7 +58,9 @@ describe('elasticsearch state storage api schema', () => { await harness.initialize(); const { apis } = harness.executionContext.config; - return apis.find((settings) => settings._name === apiName) as ESStateStorageConfig; + return apis.find( + (settings: APIConfig) => settings._name === apiName + ) as ESStateStorageConfig; } afterEach(async () => { diff --git a/test/fixtures/data/even-spread-extra1.ts b/test/fixtures/data/even-spread-extra1.ts index 8d631ab1d..8868b3e77 100644 --- a/test/fixtures/data/even-spread-extra1.ts +++ b/test/fixtures/data/even-spread-extra1.ts @@ -64,8 +64,8 @@ const types: DataTypeFields = { bytes: { type: FieldType.Integer } }; -export = { +export default { data: cloneDeep(data), types, index: 'even_spread' -} +}; diff --git a/test/helpers/elasticsearch.ts b/test/helpers/elasticsearch.ts index 2685daba0..b7b7c48d7 100644 --- a/test/helpers/elasticsearch.ts +++ b/test/helpers/elasticsearch.ts @@ -14,9 +14,6 @@ const { const logger = debugLogger('elasticsearch_helpers'); -// automatically set the timeout to 30s when using elasticsearch -jest.setTimeout(30000); - export { makeClient, formatUploadData, upload, populateIndex, waitForData, cleanupIndex, diff --git a/test/helpers/index.ts b/test/helpers/index.ts index c783d7f68..9de5c4127 100644 --- a/test/helpers/index.ts +++ b/test/helpers/index.ts @@ -1,2 +1,2 @@ -export * from './elasticsearch'; -export * from './utils'; +export * from './elasticsearch.js'; +export * from './utils.js'; diff --git a/test/id_reader/fetcher-spec.ts b/test/id_reader/fetcher-spec.ts index 2c50dc53d..1e5211844 100644 --- a/test/id_reader/fetcher-spec.ts +++ b/test/id_reader/fetcher-spec.ts @@ -1,5 +1,5 @@ import 'jest-extended'; -import { DataEntity } from '@terascope/job-components'; +import { DataEntity, debugLogger, TestClientConfig } from '@terascope/job-components'; import { ElasticsearchTestHelpers, isOpensearch2, isElasticsearch8 } from 'elasticsearch-store'; import { JobTestHarness, newTestJobConfig } from 'teraslice-test-harness'; import { getKeyArray, IDType } from '@terascope/elasticsearch-asset-apis'; @@ -10,12 +10,12 @@ import { makeClient, cleanupIndex, populateIndex -} from '../helpers'; +} from '../helpers/index.js'; describe('id_reader fetcher', () => { const idIndex = `${TEST_INDEX_PREFIX}_id_fetcher_`; const evenSpread = ElasticsearchTestHelpers.EvenDateData; - + const logger = debugLogger('test-logger'); const docType = '_doc'; // in es5 this should be ignored const id_field_name = 'uuid'; @@ -28,7 +28,7 @@ describe('id_reader fetcher', () => { const evenIndex = makeIndex('even_spread'); let harness: JobTestHarness; - let clients: any; + let clients: TestClientConfig[]; let esClient: any; let hasTypedDoc = true; @@ -51,14 +51,16 @@ describe('id_reader fetcher', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) }, { type: 'elasticsearch-next', endpoint: 'otherConnection', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; diff --git a/test/id_reader/job-spec.ts b/test/id_reader/job-spec.ts index 1187c328a..fd1d3bb5c 100644 --- a/test/id_reader/job-spec.ts +++ b/test/id_reader/job-spec.ts @@ -1,5 +1,6 @@ import 'jest-extended'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; +import { TestClientConfig, debugLogger } from '@terascope/job-components'; import { JobTestHarness, newTestJobConfig } from 'teraslice-test-harness'; import { getKeyArray, IDType } from '@terascope/elasticsearch-asset-apis'; import { @@ -9,11 +10,12 @@ import { makeClient, cleanupIndex, populateIndex -} from '../helpers'; +} from '../helpers/index.js'; describe('id_reader job', () => { const idIndex = `${TEST_INDEX_PREFIX}_id_job_`; const docType = '_doc'; + const logger = debugLogger('test-logger'); // in es5 this should be ignored const id_field_name = 'uuid'; const evenSpread = ElasticsearchTestHelpers.EvenDateData; @@ -27,7 +29,7 @@ describe('id_reader job', () => { const evenIndex = makeIndex('even_data'); let harness: JobTestHarness; - let clients: any; + let clients: TestClientConfig[]; let esClient: any; beforeAll(async () => { @@ -46,14 +48,16 @@ describe('id_reader job', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) }, { type: 'elasticsearch-next', endpoint: 'otherConnection', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; diff --git a/test/id_reader/schema-spec.ts b/test/id_reader/schema-spec.ts index 10f8364b5..9bbf88413 100644 --- a/test/id_reader/schema-spec.ts +++ b/test/id_reader/schema-spec.ts @@ -1,19 +1,23 @@ import 'jest-extended'; import { WorkerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; -import { AnyObject, ValidatedJobConfig } from '@terascope/job-components'; -import { ESIDReaderConfig } from '../../asset/src/id_reader/interfaces'; -import { DEFAULT_API_NAME } from '../../asset/src/elasticsearch_reader_api/interfaces'; +import { + ValidatedJobConfig, OpConfig, APIConfig, + debugLogger, TestClientConfig +} from '@terascope/job-components'; +import { ESIDReaderConfig } from '../../asset/src/id_reader/interfaces.js'; +import { DEFAULT_API_NAME } from '../../asset/src/elasticsearch_reader_api/interfaces.js'; import { TEST_INDEX_PREFIX, makeClient, cleanupIndex, populateIndex -} from '../helpers'; +} from '../helpers/index.js'; describe('id_reader Schema', () => { const name = 'id_reader'; const id_field_name = 'someField'; + const logger = debugLogger('test-logger'); const readerIndex = `${TEST_INDEX_PREFIX}_elasticsearch_id_reader_schema`; @@ -31,17 +35,17 @@ describe('id_reader Schema', () => { let harness: WorkerTestHarness; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; - async function makeSchema(config: AnyObject = {}): Promise { - const base: AnyObject = {}; + async function makeSchema(config: Record = {}): Promise { + const base: Record = {}; const opConfig = Object.assign(base, { _op: name, index, id_field_name }, config); harness = WorkerTestHarness.testFetcher(opConfig, { clients }); await harness.initialize(); const validConfig = harness.executionContext.config.operations.find( - (testConfig) => testConfig._op === name + (testConfig: OpConfig) => testConfig._op === name ); return validConfig as ESIDReaderConfig; @@ -60,7 +64,8 @@ describe('id_reader Schema', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) }, ]; @@ -117,7 +122,10 @@ describe('id_reader Schema', () => { ] }); - expect(() => new WorkerTestHarness(job, { clients })).toThrow(); + await expect(async () => { + const test = new WorkerTestHarness(job, { clients }); + await test.initialize(); + }).rejects.toThrow(); }); it('should not throw if base api is created but opConfig has index set to another value', async () => { @@ -136,7 +144,7 @@ describe('id_reader Schema', () => { await harness.initialize(); const apiConfig = harness.executionContext.config.apis.find( - (api) => api._name === 'elasticsearch_reader_api:id_reader-0' + (api: APIConfig) => api._name === 'elasticsearch_reader_api:id_reader-0' ); expect(apiConfig).toMatchObject({ index }); @@ -156,7 +164,10 @@ describe('id_reader Schema', () => { ] }); - expect(() => new WorkerTestHarness(job, { clients })).toThrow(); + await expect(async () => { + const test = new WorkerTestHarness(job, { clients }); + await test.initialize(); + }).rejects.toThrow(); }); it('should throw if number of slicers are greater than key_type length', async () => { @@ -168,7 +179,10 @@ describe('id_reader Schema', () => { ] }); - expect(() => new WorkerTestHarness(job, { clients })).toThrow(); + await expect(async () => { + const test = new WorkerTestHarness(job, { clients }); + await test.initialize(); + }).rejects.toThrow(); const job2 = newTestJobConfig({ slicers: 17, @@ -178,7 +192,10 @@ describe('id_reader Schema', () => { ] }); - expect(() => new WorkerTestHarness(job2, { clients })).toThrow(); + await expect(async () => { + const test = new WorkerTestHarness(job2, { clients }); + await test.initialize(); + }).rejects.toThrow(); }); it('can validateJob to make sure its configured correctly', async () => { diff --git a/test/id_reader/slicer-spec.ts b/test/id_reader/slicer-spec.ts index b4381daed..2b5dd6644 100644 --- a/test/id_reader/slicer-spec.ts +++ b/test/id_reader/slicer-spec.ts @@ -1,5 +1,5 @@ import { SlicerTestHarness, newTestJobConfig } from 'teraslice-test-harness'; -import { SlicerRecoveryData, AnyObject } from '@terascope/job-components'; +import { SlicerRecoveryData, debugLogger, TestClientConfig } from '@terascope/job-components'; import { ElasticsearchTestHelpers } from 'elasticsearch-store'; import { TEST_INDEX_PREFIX, @@ -7,22 +7,20 @@ import { makeClient, populateIndex, waitForData, -} from '../helpers'; - -jest.setTimeout(30 * 1000); +} from '../helpers/index.js'; describe('id_reader slicer', () => { const apiReaderIndex = `${TEST_INDEX_PREFIX}_id_slicer`; const docType = '_doc'; const field = 'uuid'; const evenSpread = ElasticsearchTestHelpers.EvenDateData; - + const logger = debugLogger('test-logger'); // for compatibility tests for older elasticsearch version, // we make the _id of the record the same as its uuid field const bulkData = evenSpread.data; let harness: SlicerTestHarness; - let clients: any; + let clients: TestClientConfig[]; let esClient: any; beforeAll(async () => { @@ -42,7 +40,8 @@ describe('id_reader slicer', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }) } ]; @@ -60,7 +59,7 @@ describe('id_reader slicer', () => { }; async function makeSlicerTest( - opConfig: AnyObject = {}, + opConfig: Record = {}, numOfSlicers = 1, recoveryData: SlicerRecoveryData[]|undefined = undefined ) { diff --git a/test/spaces_reader/fetcher-spec.ts b/test/spaces_reader/fetcher-spec.ts index 589be3e4d..8bca6cc44 100644 --- a/test/spaces_reader/fetcher-spec.ts +++ b/test/spaces_reader/fetcher-spec.ts @@ -1,14 +1,18 @@ import 'jest-extended'; import nock from 'nock'; -import { DataEntity, newTestJobConfig } from '@terascope/job-components'; +import { + DataEntity, newTestJobConfig, TestClientConfig, + debugLogger +} from '@terascope/job-components'; import { WorkerTestHarness } from 'teraslice-test-harness'; -import MockClient from '../helpers/mock_client'; +import MockClient from '../helpers/mock_client.js'; describe('spaces_reader fetcher', () => { const baseUri = 'http://test.dev'; const testIndex = 'details-subset'; + const logger = debugLogger('test-logger'); - let clients: any; + let clients: TestClientConfig[]; let defaultClient: MockClient; const maxSize = 100000; @@ -21,7 +25,8 @@ describe('spaces_reader fetcher', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: defaultClient + client: defaultClient, + logger }), } ]; diff --git a/test/spaces_reader/schema-spec.ts b/test/spaces_reader/schema-spec.ts index 879110d53..3311c98e8 100644 --- a/test/spaces_reader/schema-spec.ts +++ b/test/spaces_reader/schema-spec.ts @@ -1,20 +1,21 @@ import 'jest-extended'; import { - newTestJobConfig, AnyObject + newTestJobConfig, AnyObject, APIConfig, + debugLogger, TestClientConfig } from '@terascope/job-components'; import { WorkerTestHarness } from 'teraslice-test-harness'; -import { makeClient } from '../helpers'; -import { DEFAULT_API_NAME } from '../../asset/src/spaces_reader_api/interfaces'; +import { makeClient } from '../helpers/index.js'; +import { DEFAULT_API_NAME } from '../../asset/src/spaces_reader_api/interfaces.js'; describe('spaces-reader schema', () => { let harness: WorkerTestHarness; const index = 'some_index'; const name = 'spaces_reader'; - + const logger = debugLogger('test-logger'); const docType = '_doc'; let esClient: any; - let clients: any; + let clients: TestClientConfig[]; beforeAll(async () => { esClient = await makeClient(); @@ -24,7 +25,8 @@ describe('spaces-reader schema', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: esClient + client: esClient, + logger }), } ]; @@ -107,7 +109,10 @@ describe('spaces-reader schema', () => { ] }); - expect(() => new WorkerTestHarness(job, { clients })).toThrow(); + await expect(async () => { + const test = new WorkerTestHarness(job, { clients }); + await test.initialize(); + }).rejects.toThrow(); }); it('should not throw if base api is created but opConfig has index set to another value', async () => { @@ -138,7 +143,7 @@ describe('spaces-reader schema', () => { await harness.initialize(); const apiConfig = harness.executionContext.config.apis.find( - (api) => api._name === 'spaces_reader_api:spaces_reader-0' + (api: APIConfig) => api._name === 'spaces_reader_api:spaces_reader-0' ); expect(apiConfig).toMatchObject({ index }); diff --git a/test/spaces_reader/slicer-spec.ts b/test/spaces_reader/slicer-spec.ts index 8bb1f7eeb..ac5e48100 100644 --- a/test/spaces_reader/slicer-spec.ts +++ b/test/spaces_reader/slicer-spec.ts @@ -1,19 +1,26 @@ import 'jest-extended'; import nock from 'nock'; -import path from 'path'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import moment from 'moment'; -import { newTestJobConfig, SlicerRecoveryData } from '@terascope/job-components'; +import { + newTestJobConfig, SlicerRecoveryData, debugLogger, + TestClientConfig +} from '@terascope/job-components'; import { SlicerTestHarness } from 'teraslice-test-harness'; -import MockClient from '../helpers/mock_client'; +import MockClient from '../helpers/mock_client.js'; + +const dirname = path.dirname(fileURLToPath(import.meta.url)); describe('spaces_reader slicer', () => { const baseUri = 'http://test.dev'; const testIndex = 'details-subset'; - const assetDir = path.join(__dirname, '../..'); + const assetDir = path.join(dirname, '../..'); const maxSize = 100000; const token = 'test-token'; + const logger = debugLogger('test-logger'); - let clients: any; + let clients: TestClientConfig[]; let defaultClient: MockClient; interface EventHook { @@ -28,7 +35,8 @@ describe('spaces_reader slicer', () => { type: 'elasticsearch-next', endpoint: 'default', createClient: async () => ({ - client: defaultClient + client: defaultClient, + logger }), } ]; diff --git a/test/test.setup.js b/test/test.setup.js deleted file mode 100644 index aec2bd7fc..000000000 --- a/test/test.setup.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -process.env.NODE_ENV = 'test'; - -jest.setTimeout(15000); diff --git a/tsconfig.json b/tsconfig.json index bdbac101a..f4507b741 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,9 +2,9 @@ "compilerOptions": { "baseUrl": ".", "outDir": "dist", - "module": "commonjs", "moduleResolution": "node", - "target": "es2019", + "target": "ESNext", + "module": "ESNext", "skipLibCheck": true, "experimentalDecorators": true, "strict": true, @@ -26,9 +26,6 @@ }, "include": ["test"], "references": [ - { - "path": "packages/terafoundation_elasticsearch_connector" - }, { "path": "packages/elasticsearch-asset-apis" }, diff --git a/yarn.lock b/yarn.lock index df95c32c0..53339c183 100644 --- a/yarn.lock +++ b/yarn.lock @@ -324,16 +324,6 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@elastic/elasticsearch@7.13.0": - version "7.13.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.13.0.tgz#6dcf511dfa91187e22c81e54f41f4bd0fd96b4d6" - integrity sha512-WgwLWo2p9P2tdqzBGX9fHeG8p5IOTXprXNTECQG2mJ7z9n93N5AFBJpEw4d35tWWeCWi9jI13A2wzQZH7XZ/xw== - dependencies: - debug "^4.3.1" - hpagent "^0.1.1" - ms "^2.1.3" - secure-json-parse "^2.4.0" - "@elastic/transport@^8.2.0": version "8.2.0" resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.2.0.tgz#f292cb79c918a36268dd853431e41f13544814ad" @@ -685,16 +675,6 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@opensearch-project/opensearch@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7" - integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ== - dependencies: - debug "^4.3.1" - hpagent "^0.1.1" - ms "^2.1.3" - secure-json-parse "^2.4.0" - "@opentelemetry/api@^1.4.0": version "1.8.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.8.0.tgz#5aa7abb48f23f693068ed2999ae627d2f7d902ec" @@ -710,6 +690,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== +"@sindresorhus/is@^5.2.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-5.6.0.tgz#41dd6093d34652cddb5d5bdeee04eafc33826668" + integrity sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g== + "@sinonjs/commons@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.0.tgz#beb434fe875d965265e04722ccfc21df7f755d72" @@ -731,15 +716,22 @@ dependencies: defer-to-connect "^2.0.0" -"@terascope/data-mate@^0.56.2": - version "0.56.2" - resolved "https://registry.yarnpkg.com/@terascope/data-mate/-/data-mate-0.56.2.tgz#51fa9b4ba49e5090acdb9e028e07f61ef3ac7f6d" - integrity sha512-qRidKi3UcUThdzj+y9G160vgxO4X7x1v+5MVS0+pYgyx27kTmZ6LBlrD7aYwbm6TQt6Z4oc9048iMq8pnH5bjg== +"@szmarczak/http-timer@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-5.0.1.tgz#c7c1bf1141cdd4751b0399c8fc7b8b664cd5be3a" + integrity sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw== + dependencies: + defer-to-connect "^2.0.1" + +"@terascope/data-mate@^0.56.3": + version "0.56.3" + resolved "https://registry.yarnpkg.com/@terascope/data-mate/-/data-mate-0.56.3.tgz#30f9e599681856a04f55bd53650ca36f82ed675f" + integrity sha512-+WwxIiLTi9N1Tmab2N3b0VrsD2YUnnDyFJe6D9AokMj9luA6Brv+5c58JFfzL8lhFOd3b8/HrSPTdNELgBGiIA== dependencies: - "@terascope/data-types" "^0.50.2" - "@terascope/types" "^0.17.2" - "@terascope/utils" "^0.59.2" - "@types/validator" "^13.11.9" + "@terascope/data-types" "^0.50.3" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" + "@types/validator" "^13.11.10" awesome-phonenumber "^2.70.0" date-fns "^2.30.0" ip-bigint "^3.0.3" @@ -752,27 +744,27 @@ mnemonist "^0.39.8" uuid "^9.0.1" valid-url "^1.0.9" - validator "^13.11.0" - xlucene-parser "^0.58.2" + validator "^13.12.0" + xlucene-parser "^0.58.3" -"@terascope/data-types@^0.50.1", "@terascope/data-types@^0.50.2": - version "0.50.2" - resolved "https://registry.yarnpkg.com/@terascope/data-types/-/data-types-0.50.2.tgz#63e8d8fb078e67bee82a664a9d5c608e2bc9e1f2" - integrity sha512-VNXyWCkuZQLrgT9oXys3rGsRGyh90ah9JGmAZlrguTgd02UNQEgSYZji7M7kn7SKHnFpzHHwuzqX7kwv++jXfA== +"@terascope/data-types@^0.50.3": + version "0.50.3" + resolved "https://registry.yarnpkg.com/@terascope/data-types/-/data-types-0.50.3.tgz#bda225535b40c45c3c393dcc4155159f14f3db23" + integrity sha512-i60T0nF/nKKF/f9YS/EmFmpDhdLExJfvjg28OerAXQ+eci83+VETlCa3dpsQek7NzdtZ49Fwtx+ab8AVotWeKQ== dependencies: - "@terascope/types" "^0.17.2" - "@terascope/utils" "^0.59.2" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" graphql "^14.7.0" lodash "^4.17.21" yargs "^17.7.2" -"@terascope/elasticsearch-api@^3.20.2": - version "3.20.2" - resolved "https://registry.yarnpkg.com/@terascope/elasticsearch-api/-/elasticsearch-api-3.20.2.tgz#f207d68d672bb8bd36fa8e25e6c2c7f21204a70b" - integrity sha512-Tg5EKfKuGurHo9RSl5HHhRUKGXNpflqKjq6nXwfm6A3sO3cO4nMK/TzSxX+/+Dhx8yjgU/0BKOKwCHsoU8R+tw== +"@terascope/elasticsearch-api@^3.20.3": + version "3.20.3" + resolved "https://registry.yarnpkg.com/@terascope/elasticsearch-api/-/elasticsearch-api-3.20.3.tgz#d8ae7b8c1d20bf4788c76d70b5d7375f7ce0c72b" + integrity sha512-4WfUad2Hyh5XeJXvfrAnAg7UblH6iSCJgQOVYWzQekze5tOPVG2TF3odDRq2F63tpC/cx81DP+NIGmBt/vo5vw== dependencies: - "@terascope/types" "^0.17.2" - "@terascope/utils" "^0.59.2" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" bluebird "^3.7.2" setimmediate "^1.0.5" @@ -791,38 +783,39 @@ eslint-plugin-react "^7.29.4" eslint-plugin-react-hooks "^4.4.0" -"@terascope/fetch-github-release@^0.8.7": - version "0.8.7" - resolved "https://registry.yarnpkg.com/@terascope/fetch-github-release/-/fetch-github-release-0.8.7.tgz#fd8aba265c06704d51a6ab70fd2c656c80f04156" - integrity sha512-m6vpKCUlBYhxlx6BXQoOi0hg/FFZ5Bo/De+MjhDuoFLhR8pMzJxS0Nge8bSIXc1G8Jqmw4g4mkoZOqEb9TmMbQ== +"@terascope/fetch-github-release@^0.8.10": + version "0.8.10" + resolved "https://registry.yarnpkg.com/@terascope/fetch-github-release/-/fetch-github-release-0.8.10.tgz#243b8a50f963f4c2c5771689e2b94e94265db179" + integrity sha512-bTLmiEotEca8YTi+PcvznisIHMEKYLU0Uj322rkbMJlr6LqXFvIKv2dzuwMhSADO7eUeKlC4mXUs8Nt2ftpdWA== dependencies: extract-zip "^2.0.1" - gauge "^3.0.0" got "^11.4.0" multi-progress "^4.0.0" progress "^2.0.3" yargs "^17.2.1" -"@terascope/job-components@^0.75.1": - version "0.75.1" - resolved "https://registry.yarnpkg.com/@terascope/job-components/-/job-components-0.75.1.tgz#74cdd3035414ce56a9762403afc797e0a41827be" - integrity sha512-h8fYs3vBRBHf90q0JJhjNi/QeaJ3whsqNIgu53WnBWrHrlxP3Nfj9bYxoDFf7Ne4w31lDI/6Z5Z69tutEQQ3mA== +"@terascope/job-components@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@terascope/job-components/-/job-components-1.0.1.tgz#8b271a17550aad9d01baa3d9584e6a35adf3d866" + integrity sha512-MsbzVnQf87LkzDI7b75RGBmRrMqchqrCEdBacO0PwJ9OakFWeqmJv8UDolTlRe3zfb3OgfWwhSK7V2jpvdZ+CA== dependencies: - "@terascope/utils" "^0.59.2" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" convict "^6.2.4" convict-format-with-moment "^6.2.0" convict-format-with-validator "^6.2.0" datemath-parser "^1.0.6" + import-meta-resolve "^4.0.0" prom-client "^15.1.2" uuid "^9.0.1" -"@terascope/scripts@0.77.2": - version "0.77.2" - resolved "https://registry.yarnpkg.com/@terascope/scripts/-/scripts-0.77.2.tgz#4f6bea89928d6dfa754a685f8ad4e55e86ce7902" - integrity sha512-XWQTYhz4qYQ5N5zc3P0WQNktnwE6lMuqFHn+4WvU8wayBJtJUp6Ljvq9MTXnf7tkTclFJwBQZv+7flOwuRueiA== +"@terascope/scripts@0.77.3": + version "0.77.3" + resolved "https://registry.yarnpkg.com/@terascope/scripts/-/scripts-0.77.3.tgz#8d3b2f6271527aab336b86a64efac796f2819018" + integrity sha512-y89ygMBL4hjUVKo2r2YrODqEog3SanYi+zLQes1Tlhu5y2qN/mw4jxZlktajGWNClT7vnknp++hlf09JPJIdyw== dependencies: "@kubernetes/client-node" "^0.20.0" - "@terascope/utils" "^0.59.2" + "@terascope/utils" "^0.59.3" codecov "^3.8.3" execa "^5.1.0" fs-extra "^11.2.0" @@ -837,7 +830,7 @@ ms "^2.1.3" package-json "^7.0.0" pkg-up "^3.1.0" - semver "^7.6.1" + semver "^7.6.2" signale "^1.4.0" sort-package-json "~1.57.0" toposort "^2.0.2" @@ -852,27 +845,27 @@ dependencies: bluebird "^3.7.2" -"@terascope/teraslice-state-storage@^0.53.2": - version "0.53.2" - resolved "https://registry.yarnpkg.com/@terascope/teraslice-state-storage/-/teraslice-state-storage-0.53.2.tgz#bc21d9b5fd9bade3e82121ae2c0d6c69a67ba1c0" - integrity sha512-6uG/kTodEYNyWopfTHA2zMqr488HmZlryEIPa2eBGxZO5vr46cjnd1EOcY5gDDfdSruVQMwq2z++SFTv4+RgTw== +"@terascope/teraslice-state-storage@^0.53.3": + version "0.53.3" + resolved "https://registry.yarnpkg.com/@terascope/teraslice-state-storage/-/teraslice-state-storage-0.53.3.tgz#fb8679d6a3407122b0d8191c5775ba59bcb4da36" + integrity sha512-M7F9AnD+1mCJVzYEEvi5kBC0m7raA27OxtTNi7sLzny8Zp1KtEAq6SKgkPRndEjkQIY7CuaQozIi7nNPvNzyYA== dependencies: - "@terascope/elasticsearch-api" "^3.20.2" - "@terascope/utils" "^0.59.2" + "@terascope/elasticsearch-api" "^3.20.3" + "@terascope/utils" "^0.59.3" -"@terascope/types@^0.17.1", "@terascope/types@^0.17.2": - version "0.17.2" - resolved "https://registry.yarnpkg.com/@terascope/types/-/types-0.17.2.tgz#0e8d13c1249dccb0ccf63e691e253337645a205d" - integrity sha512-GzbxIVjFiViiULc7XK0EiAgT5yubwi2VXvp0wC3ygigTNIsqugncPub2ChpiqTx2JlVGVGJrXPSvPgPTJphYqw== +"@terascope/types@^0.17.3": + version "0.17.3" + resolved "https://registry.yarnpkg.com/@terascope/types/-/types-0.17.3.tgz#a99529ed1dd3aeb28a667e19749e0c2bb34f775f" + integrity sha512-4HtoBwWFcOMvtY1khQTalQIu6EfiQNKFJlagJfL7EWrqa1p968tS0NaqpZD8foIHamVEFQ5m0b4TgJeYHjzUbg== dependencies: prom-client "^15.1.2" -"@terascope/utils@^0.59.1", "@terascope/utils@^0.59.2": - version "0.59.2" - resolved "https://registry.yarnpkg.com/@terascope/utils/-/utils-0.59.2.tgz#0c987c2d307a19abe7e41036c162bc382df741a6" - integrity sha512-CYEfVt7PkkKFdpnnMf0sXfd6dYQdK3SsrKkCuupYvk3nPyP5q1eRzyzX4IS1/KrG3yVsqTKNHLrx8gR/UHUPOw== +"@terascope/utils@^0.59.3": + version "0.59.3" + resolved "https://registry.yarnpkg.com/@terascope/utils/-/utils-0.59.3.tgz#24e91b25833abc324f6038664d6d567fffbe79df" + integrity sha512-q1MFXLe6sfJ7Eybraz90RfI9YnrHSFz9SyWmqIV3yrqyT/+OAzwZ2wy9fX2dHqA6WP69t0LAD1J/n/9Og6JSlg== dependencies: - "@terascope/types" "^0.17.2" + "@terascope/types" "^0.17.3" "@turf/bbox" "^6.4.0" "@turf/bbox-polygon" "^6.4.0" "@turf/boolean-contains" "^6.4.0" @@ -886,7 +879,7 @@ "@turf/invariant" "^6.2.0" "@turf/line-to-polygon" "^6.4.0" "@types/lodash" "^4.14.202" - "@types/validator" "^13.11.9" + "@types/validator" "^13.11.10" awesome-phonenumber "^2.70.0" date-fns "^2.30.0" date-fns-tz "^1.3.7" @@ -907,7 +900,7 @@ mnemonist "^0.39.8" p-map "^4.0.0" shallow-clone "^3.0.1" - validator "^13.11.0" + validator "^13.12.0" "@tootallnate/once@1": version "1.1.2" @@ -1138,7 +1131,7 @@ resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.5.tgz#db9468cb1b1b5a925b8f34822f1669df0c5472f5" integrity sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg== -"@types/elasticsearch@^5.0.40", "@types/elasticsearch@^5.0.43": +"@types/elasticsearch@^5.0.43": version "5.0.43" resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.43.tgz#a3bbf56922de2d0e24c6117e8de1c9b50029c3c6" integrity sha512-N+MpzURpDCWd7zaJ7CE1aU+nBSeAABLhDE0lGodQ0LLftx7ku6hjTXLr9OAFZLSXiWL3Xxx8jts485ynrcm5NA== @@ -1177,6 +1170,11 @@ resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz#abe102d06ccda1efdf0ed98c10ccf7f36a785a41" integrity sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw== +"@types/http-cache-semantics@^4.0.2": + version "4.0.4" + resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz#b979ebad3919799c979b17c72621c0bc0a31c6c4" + integrity sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA== + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" @@ -1248,13 +1246,6 @@ dependencies: undici-types "~5.26.4" -"@types/node@^18.14.2": - version "18.18.9" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.9.tgz#5527ea1832db3bba8eb8023ce8497b7d3f299592" - integrity sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ== - dependencies: - undici-types "~5.26.4" - "@types/node@^20.1.1": version "20.10.3" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.3.tgz#4900adcc7fc189d5af5bb41da8f543cea6962030" @@ -1262,6 +1253,13 @@ dependencies: undici-types "~5.26.4" +"@types/node@^20.14.6": + version "20.14.6" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.6.tgz#f3c19ffc98c2220e18de259bb172dd4d892a6075" + integrity sha512-JbA0XIJPL1IiNnU7PFxDXyfAwcwVVrOoqyzzyQTyMeVhBzkJVMSkC1LlVsRQ2lpqiY4n6Bb9oCS6lzDKVQxbZw== + dependencies: + undici-types "~5.26.4" + "@types/request@^2.47.1": version "2.48.12" resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.12.tgz#0f590f615a10f87da18e9790ac94c29ec4c5ef30" @@ -1294,10 +1292,10 @@ resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== -"@types/validator@^13.11.9": - version "13.11.9" - resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.11.9.tgz#adfe96520b437a0eaa798a475877bf2f75ee402d" - integrity sha512-FCTsikRozryfayPuiI46QzH3fnrOoctTjvOYZkho9BTFLCOZ2rgZJHMOVgCOfttjPJcgOx52EpkY0CMfy87MIw== +"@types/validator@^13.11.10": + version "13.12.0" + resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.12.0.tgz#1fe4c3ae9de5cf5193ce64717c99ef2fa7d8756f" + integrity sha512-nH45Lk7oPIJ1RVOF6JgFI6Dy0QpHEzq4QecZhvguxYPDwT8c93prCMqAtiIttm39voZ+DDR+qkNnMpJmMBRqag== "@types/ws@^8.5.3": version "8.5.10" @@ -1431,13 +1429,6 @@ agent-base@6: dependencies: debug "4" -agentkeepalive@^3.4.1: - version "3.5.2" - resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-3.5.2.tgz#a113924dd3fa24a0bc3b78108c450c2abee00f67" - integrity sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ== - dependencies: - humanize-ms "^1.2.1" - aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" @@ -1463,11 +1454,6 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.21.3" -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - integrity sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA== - ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -1478,11 +1464,6 @@ ansi-sequence-parser@^1.1.0: resolved "https://registry.yarnpkg.com/ansi-sequence-parser/-/ansi-sequence-parser-1.1.0.tgz#4d790f31236ac20366b23b3916b789e1bde39aed" integrity sha512-lEm8mt52to2fT8GhciPCGeCXACSz2UwIN4X2e2LJSnZ5uAbn2/dsYdOmUXq0AtWS5cpAupysIneExOgH0Vd2TQ== -ansi-styles@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" - integrity sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA== - ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -1510,11 +1491,6 @@ anymatch@^3.0.3: normalize-path "^3.0.0" picomatch "^2.0.4" -"aproba@^1.0.3 || ^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.0.0.tgz#52520b8ae5b569215b354efc0caa3fe1e45a8adc" - integrity sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ== - argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -1870,6 +1846,24 @@ cacheable-lookup@^5.0.3: resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005" integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA== +cacheable-lookup@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz#3476a8215d046e5a3202a9209dd13fec1f933a27" + integrity sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w== + +cacheable-request@^10.2.8: + version "10.2.14" + resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-10.2.14.tgz#eb915b665fda41b79652782df3f553449c406b9d" + integrity sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ== + dependencies: + "@types/http-cache-semantics" "^4.0.2" + get-stream "^6.0.1" + http-cache-semantics "^4.1.1" + keyv "^4.5.3" + mimic-response "^4.0.0" + normalize-url "^8.0.0" + responselike "^3.0.0" + cacheable-request@^7.0.2: version "7.0.4" resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.4.tgz#7a33ebf08613178b403635be7b899d3e69bbe817" @@ -1936,25 +1930,6 @@ caseless@~0.12.0: resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== -casual-browserify@^1.5.19: - version "1.5.19" - resolved "https://registry.yarnpkg.com/casual-browserify/-/casual-browserify-1.5.19.tgz#5c40025225d543bfeb9e19f99f453b08cd0578e1" - integrity sha512-j3xhfvYQRWxxaYDr2jMwH4xBuTd9u3ZVsPR59PYJ8MX1TX4Aw1TipxZ1r1iYDmhauqBj1AmvH6vQcku2GvQhpQ== - dependencies: - mersenne-twister "^1.0.1" - moment "^2.15.2" - -chalk@^1.0.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" - integrity sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A== - dependencies: - ansi-styles "^2.2.1" - escape-string-regexp "^1.0.2" - has-ansi "^2.0.0" - strip-ansi "^3.0.0" - supports-color "^2.0.0" - chalk@^2.3.2, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1972,11 +1947,6 @@ chalk@^4.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" -chance@^1.1.7: - version "1.1.8" - resolved "https://registry.yarnpkg.com/chance/-/chance-1.1.8.tgz#5d6c2b78c9170bf6eb9df7acdda04363085be909" - integrity sha512-v7fi5Hj2VbR6dJEGRWLmJBA83LJMS47pkAbmROFxHWd9qmE1esHRZW8Clf1Fhzr3rjxnNZVCjOEv/ivFxeIMtg== - char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" @@ -2070,11 +2040,6 @@ color-name@~1.1.4: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -color-support@^1.1.2: - version "1.1.3" - resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" - integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== - combined-stream@^1.0.6, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" @@ -2107,11 +2072,6 @@ confusing-browser-globals@^1.0.10: resolved "https://registry.yarnpkg.com/confusing-browser-globals/-/confusing-browser-globals-1.0.11.tgz#ae40e9b57cdd3915408a2805ebd3a5585608dc81" integrity sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA== -console-control-strings@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" - integrity sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ== - convert-source-map@^1.6.0: version "1.9.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f" @@ -2332,7 +2292,7 @@ deepmerge@^4.2.2: resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== -defer-to-connect@^2.0.0: +defer-to-connect@^2.0.0, defer-to-connect@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587" integrity sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg== @@ -2404,11 +2364,6 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" -drange@^1.0.2: - version "1.1.1" - resolved "https://registry.yarnpkg.com/drange/-/drange-1.1.1.tgz#b2aecec2aab82fcef11dbbd7b9e32b83f8f6c0b8" - integrity sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA== - ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -2417,15 +2372,15 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -elasticsearch-store@^0.84.2: - version "0.84.2" - resolved "https://registry.yarnpkg.com/elasticsearch-store/-/elasticsearch-store-0.84.2.tgz#9c1df4091c3dc34640b750a90c8cc93c7ed07f30" - integrity sha512-hIWZmkDQn4FCQEwVIRIcXPToqlGblcS5rcMe1sQgCTnLgIi9LBfCdnGf3elFKjhygH6t7bPExxXiJliQzJQCnw== +elasticsearch-store@^0.84.3: + version "0.84.3" + resolved "https://registry.yarnpkg.com/elasticsearch-store/-/elasticsearch-store-0.84.3.tgz#b79938c98d28d1ad136061cd6fa730b2064210c8" + integrity sha512-eCgBvKssMAwSvO7w8qTV10LCH7rLXXTEP0jQwB+YblvA0Nu2cYJr7HpUhIalSq4ZK3CA0OV5SDqMvPaw4L2xCA== dependencies: - "@terascope/data-mate" "^0.56.2" - "@terascope/data-types" "^0.50.2" - "@terascope/types" "^0.17.2" - "@terascope/utils" "^0.59.2" + "@terascope/data-mate" "^0.56.3" + "@terascope/data-types" "^0.50.3" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" ajv "^6.12.6" elasticsearch6 "npm:@elastic/elasticsearch@^6.7.0" elasticsearch7 "npm:@elastic/elasticsearch@^7.0.0" @@ -2434,7 +2389,7 @@ elasticsearch-store@^0.84.2: opensearch2 "npm:@opensearch-project/opensearch@^2.2.1" setimmediate "^1.0.5" uuid "^9.0.1" - xlucene-translator "^0.44.2" + xlucene-translator "^0.44.3" "elasticsearch6@npm:@elastic/elasticsearch@^6.7.0": version "6.8.8" @@ -2467,15 +2422,6 @@ elasticsearch-store@^0.84.2: "@elastic/transport" "^8.2.0" tslib "^2.4.0" -elasticsearch@^15.4.1: - version "15.5.0" - resolved "https://registry.yarnpkg.com/elasticsearch/-/elasticsearch-15.5.0.tgz#62d9f238cf9e10b81c50a2303aa470c1f7da6b37" - integrity sha512-ZGKKaDkOFAap61ObBNkAxhYXCcAbRfkI4NVoSeLGnTD6/cItvY2j9LII/VV8/zclGe1x5m6DsVp47E4ze4aAeQ== - dependencies: - agentkeepalive "^3.4.1" - chalk "^1.0.0" - lodash "^4.17.10" - electron-to-chromium@^1.4.530: version "1.4.534" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.534.tgz#2056c1fc41a7157cdd5c634f96e758d727b69201" @@ -2647,7 +2593,7 @@ escalade@^3.1.1: resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== -escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: +escape-string-regexp@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== @@ -2933,11 +2879,6 @@ extsprintf@^1.2.0: resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.1.tgz#8d172c064867f235c0c84a596806d279bf4bcc07" integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== -faker@^5.3.1: - version "5.5.3" - resolved "https://registry.yarnpkg.com/faker/-/faker-5.5.3.tgz#c57974ee484431b25205c2c8dc09fda861e51e0e" - integrity sha512-wLTv2a28wjUyWkbnX7u/ABZBkUkIF2fCd73V6P2oFqEGEktDfzWx4UxrSqtPRw0xPRAcjeAOIiJWqZm3pP4u3g== - fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -3091,6 +3032,11 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== +form-data-encoder@^2.1.2: + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-2.1.4.tgz#261ea35d2a70d48d30ec7a9603130fa5515e9cd5" + integrity sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw== + form-data@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" @@ -3178,21 +3124,6 @@ functions-have-names@^1.2.2, functions-have-names@^1.2.3: resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== -gauge@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-3.0.2.tgz#03bf4441c044383908bcfa0656ad91803259b395" - integrity sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q== - dependencies: - aproba "^1.0.3 || ^2.0.0" - color-support "^1.1.2" - console-control-strings "^1.0.0" - has-unicode "^2.0.1" - object-assign "^4.1.1" - signal-exit "^3.0.0" - string-width "^4.2.3" - strip-ansi "^6.0.1" - wide-align "^1.1.2" - gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -3290,7 +3221,7 @@ get-stream@^5.1.0: dependencies: pump "^3.0.0" -get-stream@^6.0.0: +get-stream@^6.0.0, get-stream@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== @@ -3410,6 +3341,23 @@ got@^11.4.0, got@^11.8.2, got@^11.8.3: p-cancelable "^2.0.0" responselike "^2.0.0" +got@^13.0.0: + version "13.0.0" + resolved "https://registry.yarnpkg.com/got/-/got-13.0.0.tgz#a2402862cef27a5d0d1b07c0fb25d12b58175422" + integrity sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA== + dependencies: + "@sindresorhus/is" "^5.2.0" + "@szmarczak/http-timer" "^5.0.1" + cacheable-lookup "^7.0.0" + cacheable-request "^10.2.8" + decompress-response "^6.0.0" + form-data-encoder "^2.1.2" + get-stream "^6.0.1" + http2-wrapper "^2.1.10" + lowercase-keys "^3.0.0" + p-cancelable "^3.0.0" + responselike "^3.0.0" + graceful-fs@^4.1.10, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.10" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" @@ -3462,13 +3410,6 @@ har-validator@~5.1.3: ajv "^6.12.3" har-schema "^2.0.0" -has-ansi@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" - integrity sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg== - dependencies: - ansi-regex "^2.0.0" - has-bigints@^1.0.1, has-bigints@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -3508,11 +3449,6 @@ has-tostringtag@^1.0.0: dependencies: has-symbols "^1.0.3" -has-unicode@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" - integrity sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ== - has@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/has/-/has-1.0.4.tgz#2eb2860e000011dae4f1406a86fe80e530fb2ec6" @@ -3545,7 +3481,7 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== -http-cache-semantics@^4.0.0: +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -3576,6 +3512,14 @@ http2-wrapper@^1.0.0-beta.5.2: quick-lru "^5.1.1" resolve-alpn "^1.0.0" +http2-wrapper@^2.1.10: + version "2.2.1" + resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-2.2.1.tgz#310968153dcdedb160d8b72114363ef5fce1f64a" + integrity sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ== + dependencies: + quick-lru "^5.1.1" + resolve-alpn "^1.2.0" + https-proxy-agent@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" @@ -3589,13 +3533,6 @@ human-signals@^2.1.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== -humanize-ms@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" - integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== - dependencies: - ms "^2.0.0" - ieee754@^1.1.12, ieee754@^1.1.13: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -3629,6 +3566,11 @@ import-local@^3.0.2: pkg-dir "^4.2.0" resolve-cwd "^3.0.0" +import-meta-resolve@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz#f9db8bead9fafa61adb811db77a2bf22c5399706" + integrity sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw== + imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" @@ -4697,7 +4639,7 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash@^4.17.10, lodash@^4.17.21: +lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -4714,6 +4656,11 @@ lowercase-keys@^2.0.0: resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== +lowercase-keys@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-3.0.0.tgz#c5e7d442e37ead247ae9db117a9d0a467c89d4f2" + integrity sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ== + lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -4774,11 +4721,6 @@ merge2@^1.2.3, merge2@^1.3.0, merge2@^1.4.1: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== -mersenne-twister@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/mersenne-twister/-/mersenne-twister-1.1.0.tgz#f916618ee43d7179efcf641bec4531eb9670978a" - integrity sha512-mUYWsMKNrm4lfygPkL3OfGzOPTR2DBlTkBNHM//F6hGp8cLThY897crAlk3/Jo17LEOOjQUrNAx6DvgO77QJkA== - micromatch@^4.0.4, micromatch@^4.0.5: version "4.0.5" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" @@ -4819,6 +4761,11 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== +mimic-response@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-4.0.0.tgz#35468b19e7c75d10f5165ea25e75a5ceea7cf70f" + integrity sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg== + minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -4870,18 +4817,7 @@ mnemonist@^0.39.8: dependencies: obliterator "^2.0.1" -mocker-data-generator@^2.12.0: - version "2.12.0" - resolved "https://registry.yarnpkg.com/mocker-data-generator/-/mocker-data-generator-2.12.0.tgz#12618f095371f6167bbec234a71fbc8f2755e9e9" - integrity sha512-TE+JYb46On9xzTruZEO0Y1sgowE7C/fM4X+lWuBpCztDv9AjSnztnuAkcAgf+nH45DtT4Z9viLehhtXCps8wvA== - dependencies: - casual-browserify "^1.5.19" - chance "^1.1.7" - faker "^5.3.1" - randexp "^0.5.3" - tslib "^2.1.0" - -moment@^2.15.2, moment@^2.22.2, moment@^2.29.1, moment@^2.30.1: +moment@^2.22.2, moment@^2.29.1, moment@^2.30.1: version "2.30.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae" integrity sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how== @@ -4896,7 +4832,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -4969,6 +4905,11 @@ normalize-url@^6.0.1: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +normalize-url@^8.0.0: + version "8.0.1" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-8.0.1.tgz#9b7d96af9836577c58f5883e939365fa15623a4a" + integrity sha512-IO9QvjUMWxPQQhs60oOu10CRkWCiZzSUkzbXGGV9pviYl1fXYcvkzQ5jV9z8Y6un8ARoVRl4EtC6v6jNqbaJ/w== + npm-run-path@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" @@ -5150,6 +5091,11 @@ p-cancelable@^2.0.0: resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg== +p-cancelable@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-3.0.0.tgz#63826694b54d61ca1c20ebcb6d3ecf5e14cd8050" + integrity sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw== + p-is-promise@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971" @@ -5483,14 +5429,6 @@ quickselect@^2.0.0: resolved "https://registry.yarnpkg.com/quickselect/-/quickselect-2.0.0.tgz#f19680a486a5eefb581303e023e98faaf25dd018" integrity sha512-RKJ22hX8mHe3Y6wH/N3wCM6BWtjaxIyyUIkpHOvfFnxdI4yD4tBXEBKSbriGujF6jnSVkJrffuo6vxACiSSxIw== -randexp@^0.5.3: - version "0.5.3" - resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.5.3.tgz#f31c2de3148b30bdeb84b7c3f59b0ebb9fec3738" - integrity sha512-U+5l2KrcMNOUPYvazA3h5ekF80FHTUG+87SEAmHZmolh1M+i/WyTCxVzmi+tidIa1tM4BSe8g2Y/D3loWDjj+w== - dependencies: - drange "^1.0.2" - ret "^0.2.0" - rbush@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/rbush/-/rbush-3.0.1.tgz#5fafa8a79b3b9afdfe5008403a720cc1de882ecf" @@ -5635,7 +5573,7 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== -resolve-alpn@^1.0.0: +resolve-alpn@^1.0.0, resolve-alpn@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" integrity sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g== @@ -5703,10 +5641,12 @@ responselike@^2.0.0: dependencies: lowercase-keys "^2.0.0" -ret@^0.2.0: - version "0.2.2" - resolved "https://registry.yarnpkg.com/ret/-/ret-0.2.2.tgz#b6861782a1f4762dce43402a71eb7a283f44573c" - integrity sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ== +responselike@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/responselike/-/responselike-3.0.0.tgz#20decb6c298aff0dbee1c355ca95461d42823626" + integrity sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg== + dependencies: + lowercase-keys "^3.0.0" reusify@^1.0.4: version "1.0.4" @@ -5783,7 +5723,7 @@ semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.3.5, semver@^7.3.7, semver@^7.5.3, semver@^7.5.4, semver@^7.6.1: +semver@^7.3.5, semver@^7.3.7, semver@^7.5.3, semver@^7.5.4, semver@^7.6.2: version "7.6.2" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13" integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w== @@ -5891,7 +5831,7 @@ side-channel@^1.0.4: get-intrinsic "^1.0.2" object-inspect "^1.9.0" -signal-exit@^3.0.0, signal-exit@^3.0.3, signal-exit@^3.0.7: +signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== @@ -6007,7 +5947,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -6090,13 +6030,6 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -strip-ansi@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - integrity sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg== - dependencies: - ansi-regex "^2.0.0" - strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" @@ -6141,11 +6074,6 @@ stubs@^3.0.0: resolved "https://registry.yarnpkg.com/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b" integrity sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw== -supports-color@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" - integrity sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g== - supports-color@^5.3.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" @@ -6215,12 +6143,12 @@ teeny-request@7.1.1: stream-events "^1.0.5" uuid "^8.0.0" -teraslice-test-harness@^0.30.0: - version "0.30.0" - resolved "https://registry.yarnpkg.com/teraslice-test-harness/-/teraslice-test-harness-0.30.0.tgz#76ee0a3d06ecc3711de16edf58827dc0979b2a46" - integrity sha512-iIY2z2da7ZfzUl9bMXXQy1LwZpOfe2ICGY1fYQmiOl8UfojO4n/AENXHRIgTw2MOj875jwTDUQV+de38RwRdnA== +teraslice-test-harness@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/teraslice-test-harness/-/teraslice-test-harness-1.0.1.tgz#9612a99c66118af63036e4a6fc4a57c58b72a470" + integrity sha512-4dmMHefx7L9VLIF/Yj42QbrggkfHkZtKHI9jZNF0y5dlqRgJ2S7MhuSIq65a6ArjmGGW3XwMKjYWvY8dsiVbIQ== dependencies: - "@terascope/fetch-github-release" "^0.8.7" + "@terascope/fetch-github-release" "^0.8.10" "@terascope/teraslice-op-test-harness" "^1.24.1" decompress "^4.2.1" fs-extra "^11.2.0" @@ -6318,7 +6246,7 @@ tslib@^1.8.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2.1.0, tslib@^2.4.0: +tslib@^2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== @@ -6531,7 +6459,12 @@ valid-url@^1.0.9: resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200" integrity sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA== -validator@^13.11.0, validator@^13.6.0: +validator@^13.12.0: + version "13.12.0" + resolved "https://registry.yarnpkg.com/validator/-/validator-13.12.0.tgz#7d78e76ba85504da3fee4fd1922b385914d4b35f" + integrity sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg== + +validator@^13.6.0: version "13.11.0" resolved "https://registry.yarnpkg.com/validator/-/validator-13.11.0.tgz#23ab3fd59290c61248364eabf4067f04955fbb1b" integrity sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ== @@ -6632,13 +6565,6 @@ which@^2.0.1, which@^2.0.2: dependencies: isexe "^2.0.0" -wide-align@^1.1.2: - version "1.1.5" - resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.5.tgz#df1d4c206854369ecf3c9a4898f1b23fbd9d15d3" - integrity sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg== - dependencies: - string-width "^1.0.2 || 2 || 3 || 4" - wordwrap@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" @@ -6671,23 +6597,23 @@ ws@^8.11.0: resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f" integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g== -xlucene-parser@^0.58.2: - version "0.58.2" - resolved "https://registry.yarnpkg.com/xlucene-parser/-/xlucene-parser-0.58.2.tgz#233aa064cbac512c7a0740d62b20e29a152cf19c" - integrity sha512-smyUVk/L5i9+QheLESLzJzP28PzeWeEDIRIEsNTTrrMqQA2RqHO+3iyghr7V1ReXUG18+mun9AMP7EwS9uAinA== +xlucene-parser@^0.58.3: + version "0.58.3" + resolved "https://registry.yarnpkg.com/xlucene-parser/-/xlucene-parser-0.58.3.tgz#0b53d8c5b4617f8648d64c5b965a668a152783b2" + integrity sha512-0pFlUeP7ox2rKnYkFloyODlt9d52RDiBoPCCmaBT2/TjJIdcxSSJWemuJbdi4Vt0s87QQYbmvYvv9FUm56D8jA== dependencies: - "@terascope/types" "^0.17.2" - "@terascope/utils" "^0.59.2" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" -xlucene-translator@^0.44.2: - version "0.44.2" - resolved "https://registry.yarnpkg.com/xlucene-translator/-/xlucene-translator-0.44.2.tgz#fe181851cc9ada04728572ae5ff300cb82419c16" - integrity sha512-Gg8ec1jQxt4zdWCyjnZVPVlJI6irzkaZoFX5AUCjpnTVeJct/2GCPLaR1XrUdcupizjiUN3674Zb2kvUzcVI8A== +xlucene-translator@^0.44.3: + version "0.44.3" + resolved "https://registry.yarnpkg.com/xlucene-translator/-/xlucene-translator-0.44.3.tgz#c632a32b4619d1b2d7b65e393287a5d9e1e2e1ba" + integrity sha512-8dIjQcHZMqhNlII93KfP5ELmCMbssiAEkGQAxDhAu3VsUKrBGCip8mRfNQqxwCbetYHqo7Spl+yQjjSAbaRk7g== dependencies: - "@terascope/types" "^0.17.2" - "@terascope/utils" "^0.59.2" + "@terascope/types" "^0.17.3" + "@terascope/utils" "^0.59.3" "@types/elasticsearch" "^5.0.43" - xlucene-parser "^0.58.2" + xlucene-parser "^0.58.3" xtend@^4.0.0: version "4.0.2"