From f07e66629e04b7f14cf4b160f107b04516e0daf8 Mon Sep 17 00:00:00 2001 From: busma13 Date: Wed, 18 Dec 2024 16:33:02 -0700 Subject: [PATCH 01/21] fix errors in data-mate and elasticsearch-store --- packages/data-mate/src/index.ts | 3 +- packages/data-mate/tsconfig.json | 3 +- .../method-helpers/search.ts | 2 +- .../elasticsearch-store/src/index-model.ts | 64 +++++++++++++------ .../elasticsearch-store/src/interfaces.ts | 6 +- .../src/utils/elasticsearch.ts | 2 +- .../elasticsearch-store/src/utils/model.ts | 13 ++-- .../test/index-manager-index-setup-spec.ts | 62 ++++++++++++++---- packages/elasticsearch-store/tsconfig.json | 3 +- .../types/src/elasticsearch-interfaces.ts | 2 + 10 files changed, 114 insertions(+), 46 deletions(-) diff --git a/packages/data-mate/src/index.ts b/packages/data-mate/src/index.ts index ea89343d350..51a12e69c04 100644 --- a/packages/data-mate/src/index.ts +++ b/packages/data-mate/src/index.ts @@ -29,6 +29,7 @@ declare module './aggregation-frame/AggregationFrame.d.ts' { * @returns the new columns */ run(): Promise>; + [key: string]: any; // fixMe: make sure this is okay } } @@ -57,7 +58,7 @@ AggregationFrame.prototype.run = async function run() { const aggregationFrameMethods = Reflect.ownKeys(AggregationFrame.prototype); for (const dataFrameMethod of Reflect.ownKeys(DataFrame.prototype)) { if (!aggregationFrameMethods.includes(dataFrameMethod)) { - AggregationFrame.prototype[dataFrameMethod] = () => { + AggregationFrame.prototype[dataFrameMethod as keyof AggregationFrame] = () => { throw new Error( `Unsupported method on ${String(dataFrameMethod)} AggregationFrame. Use it before DataFrame.aggregate or after AggregationFrame.run()` diff --git a/packages/data-mate/tsconfig.json b/packages/data-mate/tsconfig.json index d0c6849850f..aae2752e13c 100644 --- a/packages/data-mate/tsconfig.json +++ b/packages/data-mate/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": "." + "rootDir": ".", + "suppressImplicitAnyIndexErrors": false }, "include": ["src", "test"] } diff --git a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/search.ts b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/search.ts index 04baefa6517..79b72fcb354 100644 --- a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/search.ts +++ b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/search.ts @@ -41,7 +41,7 @@ export function convertSearchParams( } function qDependentFieldsCheck(params: ClientParams.SearchParams) { - const requiresQ = [ + const requiresQ: Array = [ 'analyzer', 'analyze_wildcard', 'default_operator', diff --git a/packages/elasticsearch-store/src/index-model.ts b/packages/elasticsearch-store/src/index-model.ts index c9b7830efb8..33d373b2e82 100644 --- a/packages/elasticsearch-store/src/index-model.ts +++ b/packages/elasticsearch-store/src/index-model.ts @@ -1,7 +1,7 @@ import { isTest, debugLogger, concat, Logger, makeISODate, toSafeString, - trim, trimAndToLower, TSError + trim, trimAndToLower, TSError, isKey } from '@terascope/utils'; import { JoinBy } from '@terascope/data-mate'; import { QueryAccess, RestrictOptions } from 'xlucene-translator'; @@ -23,7 +23,7 @@ export abstract class IndexModel extends IndexStor readonly logger: Logger; private _uniqueFields: readonly (keyof T)[]; - private _sanitizeFields: i.SanitizeFields; + private _sanitizeFields: i.SanitizeFields | undefined; constructor( client: Client, @@ -85,7 +85,9 @@ export abstract class IndexModel extends IndexStor this.logger = options.logger || debugLogger(debugLoggerName); this._uniqueFields = concat('_key', uniqueFields); - this._sanitizeFields = sanitizeFields || {}; + if (sanitizeFields) { + this._sanitizeFields = sanitizeFields; + } this.readHooks.add((doc) => { if (doc._deleted) return false; @@ -212,24 +214,46 @@ export abstract class IndexModel extends IndexStor return count === ids.length; } + private _setStringValueOrThrow(obj: T, key: K, value: string) { + if (typeof obj[key] === 'string') { + obj[key] = value as T[K]; + } else { + throw new Error(`Cannot assign a string to a property of type ${typeof obj[key]}`); + } + } + protected _sanitizeRecord(record: T): T { - const entries = Object.entries(this._sanitizeFields); - - for (const [field, method] of entries) { - if (!record[field]) continue; - - switch (method) { - case 'trim': - record[field] = trim(record[field]); - break; - case 'trimAndToLower': - record[field] = trimAndToLower(record[field]); - break; - case 'toSafeString': - record[field] = toSafeString(record[field]); - break; - default: - continue; + if (this._sanitizeFields) { + const entries = Object.entries(this._sanitizeFields); + + for (const [field, method] of entries) { + if (isKey(record, field)) { + switch (method) { + case 'trim': + this._setStringValueOrThrow( + record, + field, + trim(record[field]) + ); + break; + case 'trimAndToLower': + this._setStringValueOrThrow( + record, + field, + trimAndToLower(record[field] as string) + ); + break; + case 'toSafeString': + this._setStringValueOrThrow( + record, + field, + toSafeString(record[field] as string) + ); + break; + default: + continue; + } + } } } diff --git a/packages/elasticsearch-store/src/interfaces.ts b/packages/elasticsearch-store/src/interfaces.ts index 73ec5a0058d..32c9f7cfa00 100644 --- a/packages/elasticsearch-store/src/interfaces.ts +++ b/packages/elasticsearch-store/src/interfaces.ts @@ -245,7 +245,7 @@ export type IndexModelConfig = Omit< unique_fields?: (keyof T)[]; /** Sanitize / cleanup fields mapping, like trim or trimAndToLower */ - sanitize_fields?: SanitizeFields; + sanitize_fields?: SanitizeFields; /** Specify whether the data should be strictly validated, defaults to true */ strict_mode?: boolean; @@ -256,8 +256,8 @@ export type IndexModelConfig = Omit< timeseries?: boolean; }; -export type SanitizeFields = { - [field: string]: 'trimAndToLower' | 'trim' | 'toSafeString'; +export type SanitizeFields = { + [field in string & keyof T]: 'trimAndToLower' | 'trim' | 'toSafeString'; }; /** diff --git a/packages/elasticsearch-store/src/utils/elasticsearch.ts b/packages/elasticsearch-store/src/utils/elasticsearch.ts index 4ad5569fe5b..605fe906f2b 100644 --- a/packages/elasticsearch-store/src/utils/elasticsearch.ts +++ b/packages/elasticsearch-store/src/utils/elasticsearch.ts @@ -246,7 +246,7 @@ export type FlattenProperties = Record): Record(source: T, from: Partial): T { +export function mergeDefaults(source: T, from: Partial): T { const output = cloneDeep(source); const _mapping = from ? cloneDeep(from) : {}; for (const [key, val] of Object.entries(_mapping)) { - if (output[key] != null) { + if (isKey(output, key)) { if (isPlainObject(val)) { - output[key] = Object.assign(output[key], val); + output[key] = Object.assign(output[key] as object, val) as T[string & keyof T]; } else if (Array.isArray(val)) { - output[key] = concat(output[key], val); + output[key] = concat(output[key], val) as T[string & keyof T]; } else { - output[key] = val; + output[key] = val as T[string & keyof T]; } } } diff --git a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts index c5ec332070a..01f960b3c23 100644 --- a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts +++ b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts @@ -1,11 +1,12 @@ import 'jest-extended'; -import { debugLogger, get } from '@terascope/utils'; +import { debugLogger, get, isKey } from '@terascope/utils'; import * as simple from './helpers/simple-index.js'; import * as template from './helpers/template-index.js'; import { IndexManager, timeSeriesIndex, IndexConfig, getESVersion, __timeSeriesTest, ElasticsearchTestHelpers } from '../src/index.js'; +import { MappingTypeMapping } from 'opensearch1/api/types.js'; const { makeClient, cleanupIndex, TEST_INDEX_PREFIX, @@ -70,7 +71,10 @@ describe('IndexManager->indexSetup()', () => { expect(mapping).toHaveProperty(index); if (esVersion === 6) { expect(mapping[index].mappings).toHaveProperty(config.name); - expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { foo: 'foo' }); + expect( + isKey(mapping[index].mappings, config.name) + && mapping[index].mappings[config.name] + ).toHaveProperty('_meta', { foo: 'foo' }); } else { expect(mapping[index].mappings).toHaveProperty('_meta', { foo: 'foo' }); } @@ -98,9 +102,18 @@ describe('IndexManager->indexSetup()', () => { it('should have updated the index metadata', async () => { const mapping = await indexManager.getMapping(index); - const properties = esVersion === 6 - ? mapping[index].mappings[config.name].properties - : mapping[index].mappings.properties; + let properties = undefined; + const { mappings } = mapping[index]; + if (esVersion === 6) { + if (isKey(mappings, config.name)) { + const mappingsObj = mappings[config.name] as MappingTypeMapping; + if (isKey(mappingsObj, 'properties')) { + properties = mappingsObj.properties; + } + } + } else { + properties = mappings.properties; + } expect(properties).toMatchObject({ test_object: { @@ -115,7 +128,10 @@ describe('IndexManager->indexSetup()', () => { }, }); if (esVersion === 6) { - expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { foo: 'foo' }); + expect( + isKey(mapping[index].mappings, config.name) + && mapping[index].mappings[config.name] + ).toHaveProperty('_meta', { foo: 'foo' }); } else { expect(mapping[index].mappings).toHaveProperty('_meta', { foo: 'foo' }); } @@ -146,9 +162,22 @@ describe('IndexManager->indexSetup()', () => { it('should have the previous the index metadata since removed fields shouldn\'t break', async () => { const mapping = await indexManager.getMapping(index); - const properties = esVersion === 6 - ? mapping[index].mappings[config.name].properties - : mapping[index].mappings.properties; + // const properties = esVersion === 6 + // ? mapping[index].mappings[config.name].properties + // : mapping[index].mappings.properties; + + let properties = undefined; + const { mappings } = mapping[index]; + if (esVersion === 6) { + if (isKey(mappings, config.name)) { + const mappingsObj = mappings[config.name] as MappingTypeMapping; + if (isKey(mappingsObj, 'properties')) { + properties = mappingsObj.properties; + } + } + } else { + properties = mappings.properties; + } expect(properties).toMatchObject({ test_object: { @@ -163,7 +192,10 @@ describe('IndexManager->indexSetup()', () => { }, }); if (esVersion === 6) { - expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { foo: 'foo' }); + expect( + isKey(mapping[index].mappings, config.name) + && mapping[index].mappings[config.name] + ).toHaveProperty('_meta', { foo: 'foo' }); } else { expect(mapping[index].mappings).toHaveProperty('_meta', { foo: 'foo' }); } @@ -229,7 +261,10 @@ describe('IndexManager->indexSetup()', () => { expect(mapping).toHaveProperty(index); if (esVersion === 6) { expect(mapping[index].mappings).toHaveProperty(config.name); - expect(mapping[index].mappings[config.name]).toHaveProperty('_meta', { bar: 'bar' }); + expect( + isKey(mapping[index].mappings, config.name) + && mapping[index].mappings[config.name] + ).toHaveProperty('_meta', { foo: 'foo' }); } else { expect(mapping[index].mappings).toHaveProperty('_meta', { bar: 'bar' }); } @@ -323,7 +358,10 @@ describe('IndexManager->indexSetup()', () => { const newIdxMapping = await indexManager.getMapping('foobar'); if (esVersion === 6) { - expect(newIdxMapping.foobar.mappings[config.name]).toHaveProperty('_meta', { baz: 'baz' }); + expect( + isKey(newIdxMapping.foobar.mappings, config.name) + && newIdxMapping.foobar.mappings[config.name] + ).toHaveProperty('_meta', { foo: 'foo' }); } else { expect(newIdxMapping.foobar.mappings).toHaveProperty('_meta', { baz: 'baz' }); } diff --git a/packages/elasticsearch-store/tsconfig.json b/packages/elasticsearch-store/tsconfig.json index d0c6849850f..aae2752e13c 100644 --- a/packages/elasticsearch-store/tsconfig.json +++ b/packages/elasticsearch-store/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": "." + "rootDir": ".", + "suppressImplicitAnyIndexErrors": false }, "include": ["src", "test"] } diff --git a/packages/types/src/elasticsearch-interfaces.ts b/packages/types/src/elasticsearch-interfaces.ts index 19bb669cdb5..d3c83789c2b 100644 --- a/packages/types/src/elasticsearch-interfaces.ts +++ b/packages/types/src/elasticsearch-interfaces.ts @@ -192,6 +192,7 @@ type BasicESTypeMapping = { type IgnoredESTypeMapping = { enabled: boolean; + [prop: string]: any; }; type FieldsESTypeMapping = { @@ -203,6 +204,7 @@ type FieldsESTypeMapping = { analyzer?: string; }; }; + [prop: string]: any; }; export type PropertyESTypes = FieldsESTypeMapping | BasicESTypeMapping; From 827fece0de2273baec5fa39bf6fced9966c13666 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Dec 2024 08:11:43 -0700 Subject: [PATCH 02/21] copy/paste error fix --- .../test/index-manager-index-setup-spec.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts index 01f960b3c23..670151a5f8d 100644 --- a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts +++ b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts @@ -162,10 +162,6 @@ describe('IndexManager->indexSetup()', () => { it('should have the previous the index metadata since removed fields shouldn\'t break', async () => { const mapping = await indexManager.getMapping(index); - // const properties = esVersion === 6 - // ? mapping[index].mappings[config.name].properties - // : mapping[index].mappings.properties; - let properties = undefined; const { mappings } = mapping[index]; if (esVersion === 6) { @@ -264,7 +260,7 @@ describe('IndexManager->indexSetup()', () => { expect( isKey(mapping[index].mappings, config.name) && mapping[index].mappings[config.name] - ).toHaveProperty('_meta', { foo: 'foo' }); + ).toHaveProperty('_meta', { bar: 'bar' }); } else { expect(mapping[index].mappings).toHaveProperty('_meta', { bar: 'bar' }); } @@ -361,7 +357,7 @@ describe('IndexManager->indexSetup()', () => { expect( isKey(newIdxMapping.foobar.mappings, config.name) && newIdxMapping.foobar.mappings[config.name] - ).toHaveProperty('_meta', { foo: 'foo' }); + ).toHaveProperty('_meta', { baz: 'baz' }); } else { expect(newIdxMapping.foobar.mappings).toHaveProperty('_meta', { baz: 'baz' }); } From 2e8e642262d8f4c6357e755245f642d38e93f5ee Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 19 Dec 2024 11:52:53 -0700 Subject: [PATCH 03/21] fix ts-transform errors --- packages/ts-transforms/src/command.ts | 9 ++++--- packages/ts-transforms/src/loader/utils.ts | 4 +-- .../ts-transforms/src/operations/index.ts | 5 ++-- .../plugins/data-mate/FieldValidator.ts | 2 +- .../plugins/data-mate/RecordTransform.ts | 2 +- .../plugins/data-mate/RecordValidator.ts | 2 +- .../src/operations/plugins/validator/index.ts | 25 +++++++++++++------ .../test/loader/rules-validator-spec.ts | 8 +++--- .../validations/validator-lib-spec.ts | 4 +-- packages/ts-transforms/tsconfig.json | 3 ++- 10 files changed, 38 insertions(+), 26 deletions(-) diff --git a/packages/ts-transforms/src/command.ts b/packages/ts-transforms/src/command.ts index 46a05f88d06..cc49b173b76 100644 --- a/packages/ts-transforms/src/command.ts +++ b/packages/ts-transforms/src/command.ts @@ -10,6 +10,7 @@ import { } from '@terascope/utils'; import { PhaseManager } from './index.js'; import { PhaseConfig } from './interfaces.js'; +import { xLuceneFieldType } from 'packages/types/dist/src/xlucene-interfaces.js'; const dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -49,7 +50,7 @@ const filePath = command.rules as string; const dataPath = command.data as string; const streamData = command.f && command.f === 'ldjson' ? command.f : false; const ignoreErrors = command.i || false; -let typesConfig = {}; +let typesConfig: Record = {}; // fixme: confirm it's always one of these const type = command.m ? 'matcher' : 'transform'; interface ESData { @@ -64,7 +65,7 @@ try { if (pieces.length !== 2) { throw new Error(`Expected -t option line #${index} to have key:value pair format, got ${segment}`); } - typesConfig[pieces[0].trim()] = pieces[1].trim(); + typesConfig[pieces[0].trim()] = pieces[1].trim() as xLuceneFieldType; }); } if (command.T) { @@ -136,10 +137,10 @@ function toJSON(obj: AnyObject) { } function handleParsedData( - data: Record[] | Record + data: Record[] ): DataEntity>[] { // input from elasticsearch - const elasticSearchResults = get(data[0], 'hits.hits', null); + const elasticSearchResults = get(data[0], 'hits.hits', null) as ESData[] | null; if (elasticSearchResults) { return elasticSearchResults.map((doc: ESData) => DataEntity.make(doc._source)); } diff --git a/packages/ts-transforms/src/loader/utils.ts b/packages/ts-transforms/src/loader/utils.ts index 0e7f1b5789c..16b4566d0c0 100644 --- a/packages/ts-transforms/src/loader/utils.ts +++ b/packages/ts-transforms/src/loader/utils.ts @@ -154,7 +154,7 @@ function findConfigs( ): FieldSourceConfigs[] { const identifier = config.follow || config.__id; const nodeIds: string[] = tagMapping[identifier]; - const mapping = {}; + const mapping: Record = {}; const results: FieldSourceConfigs[] = []; configList @@ -326,7 +326,7 @@ function createResults(list: OperationConfig[]): ValidationResults { }; const { output } = results; let currentSelector: undefined | string; - const duplicateListing = {}; + const duplicateListing: Record = {}; list.forEach((config) => { if (duplicateListing[config.__id]) { diff --git a/packages/ts-transforms/src/operations/index.ts b/packages/ts-transforms/src/operations/index.ts index 95b6907cf39..c459b447a01 100644 --- a/packages/ts-transforms/src/operations/index.ts +++ b/packages/ts-transforms/src/operations/index.ts @@ -34,7 +34,7 @@ import MacAddress from './lib/validations/mac-address.js'; import Uuid from './lib/validations/uuid.js'; import ISDN from './lib/validations/isdn.js'; import { Validator, ValidatorPlugins } from './plugins/validator/index.js'; -import dataMapePlugin from './plugins/data-mate/index.js'; +import dataMatePlugin from './plugins/data-mate/index.js'; import { OperationsDict, PluginClassType, BaseOperationClass, PluginList @@ -80,9 +80,8 @@ class OperationsManager { constructor(pluginList: PluginList = []) { pluginList.push(CorePlugins); - // @ts-expect-error FIXME: try to remove this ignore pluginList.push(ValidatorPlugins); - pluginList.push(dataMapePlugin); + pluginList.push(dataMatePlugin); const operations = pluginList.reduce((plugins, PluginClass) => { const plugin = new PluginClass(); diff --git a/packages/ts-transforms/src/operations/plugins/data-mate/FieldValidator.ts b/packages/ts-transforms/src/operations/plugins/data-mate/FieldValidator.ts index 8acc6340fc2..90948994764 100644 --- a/packages/ts-transforms/src/operations/plugins/data-mate/FieldValidator.ts +++ b/packages/ts-transforms/src/operations/plugins/data-mate/FieldValidator.ts @@ -75,7 +75,7 @@ class Validator extends OperationBase { } } -function setup(method: string) { +function setup(method: keyof typeof FieldValidator) { return InjectMethod(Validator, FieldValidator[method], FieldValidator.repository[method]); } diff --git a/packages/ts-transforms/src/operations/plugins/data-mate/RecordTransform.ts b/packages/ts-transforms/src/operations/plugins/data-mate/RecordTransform.ts index a40999b9b34..6ba506af152 100644 --- a/packages/ts-transforms/src/operations/plugins/data-mate/RecordTransform.ts +++ b/packages/ts-transforms/src/operations/plugins/data-mate/RecordTransform.ts @@ -28,7 +28,7 @@ class Transforms extends TransformsOpBase { } } -function setup(method: string) { +function setup(method: keyof typeof RecordTransform) { return InjectMethod(Transforms, RecordTransform[method], RecordTransform.repository[method]); } diff --git a/packages/ts-transforms/src/operations/plugins/data-mate/RecordValidator.ts b/packages/ts-transforms/src/operations/plugins/data-mate/RecordValidator.ts index 258dd3eab92..fe9c1cde207 100644 --- a/packages/ts-transforms/src/operations/plugins/data-mate/RecordValidator.ts +++ b/packages/ts-transforms/src/operations/plugins/data-mate/RecordValidator.ts @@ -37,7 +37,7 @@ class Validator extends OperationBase { } } -function setup(method: string) { +function setup(method: keyof typeof RecordValidator) { return InjectMethod(Validator, RecordValidator[method], RecordValidator.repository[method]); } diff --git a/packages/ts-transforms/src/operations/plugins/validator/index.ts b/packages/ts-transforms/src/operations/plugins/validator/index.ts index c251291a8e4..bf0340b947b 100644 --- a/packages/ts-transforms/src/operations/plugins/validator/index.ts +++ b/packages/ts-transforms/src/operations/plugins/validator/index.ts @@ -1,13 +1,13 @@ import { deprecate } from 'util'; import validator from 'validator'; import ValidationOpBase from '../../lib/validations/base.js'; -import { PostProcessConfig, PluginClassType, InputOutputCardinality } from '../../../interfaces.js'; +import { PostProcessConfig, PluginClassType, InputOutputCardinality, OperationsDict } from '../../../interfaces.js'; export class Validator extends ValidationOpBase { - private method: string; + private method: keyof typeof validator; private value: any; - constructor(config: PostProcessConfig, method: string) { + constructor(config: PostProcessConfig, method: keyof typeof validator) { super(config); this.method = method; this.value = config.value; @@ -15,11 +15,23 @@ export class Validator extends ValidationOpBase { validate(value: any) { const args = this.value || this.config; - return validator[this.method](value, args); + const method = validator[this.method]; + if (typeof method === 'function') { + return (method as (value: any, ...args: any[]) => any)(value, args); + } + + throw new Error(`Validator method ${this.method} is not callable`); } } -function setup(method: string) { +interface ValidatorInterfaceType { + new (config: PostProcessConfig): Validator; + cardinality: InputOutputCardinality; + +} + +function setup(method: keyof typeof validator): ValidatorInterfaceType { + // @ts-expect-error fixMe: properly type a class returning a different class return class ValidatorInterface { static cardinality: InputOutputCardinality = 'one-to-one'; @@ -30,8 +42,7 @@ function setup(method: string) { } export class ValidatorPlugins implements PluginClassType { - // @ts-expect-error - init() { + init(): OperationsDict { return { after: deprecate(setup('isAfter'), 'after is being deprecated., please use isAfter instead', 'after'), alpha: deprecate(setup('isAlpha'), 'alpha is being deprecated, please use isAlpha instead', 'alpha'), diff --git a/packages/ts-transforms/test/loader/rules-validator-spec.ts b/packages/ts-transforms/test/loader/rules-validator-spec.ts index d7909a37d8d..71f1b47b71f 100644 --- a/packages/ts-transforms/test/loader/rules-validator-spec.ts +++ b/packages/ts-transforms/test/loader/rules-validator-spec.ts @@ -377,7 +377,7 @@ describe('rules-validator', () => { it('can return an basic extractions formatted correctly', () => { const validator = constructValidator(basicExtractionConfig); const { extractions } = validator.validate(); - const results = {}; + const results: Record = {}; results['hello:world'] = basicExtractionConfig; expect(extractions).toEqual(results); @@ -386,7 +386,7 @@ describe('rules-validator', () => { it('can return an expression extractions formatted correctly', () => { const validator = constructValidator(basicExpressionConfig); const { extractions } = validator.validate(); - const results = {}; + const results: Record = {}; results['hello:world'] = basicExpressionConfig; expect(extractions).toEqual(results); @@ -395,7 +395,7 @@ describe('rules-validator', () => { it('can multiple extractions', () => { const validator = constructValidator(multiSelectorConfig); const { extractions } = validator.validate(); - const results = {}; + const results: Record = {}; results['hello:world'] = [multiSelectorConfig[0]]; results['other:things'] = [multiSelectorConfig[1]]; @@ -407,7 +407,7 @@ describe('rules-validator', () => { it('can work with newJoinRules', () => { const validator = constructValidator(newJoinRules); const { extractions } = validator.validate(); - const results = {}; + const results: Record = {}; results['hello:world'] = newJoinRules.slice(0, 2); diff --git a/packages/ts-transforms/test/operations/validations/validator-lib-spec.ts b/packages/ts-transforms/test/operations/validations/validator-lib-spec.ts index caa4eee5326..6211c355633 100644 --- a/packages/ts-transforms/test/operations/validations/validator-lib-spec.ts +++ b/packages/ts-transforms/test/operations/validations/validator-lib-spec.ts @@ -7,9 +7,9 @@ describe('validator lib', () => { const operationsClass = new ValidatorPlugins(); const operations = operationsClass.init(); - function getValidator(opConfig: PostProcessConfig, method: string): Validator { + function getValidator(opConfig: PostProcessConfig, method: keyof typeof operations): Validator { const Class = operations[method]; - return new Class(opConfig); + return new Class(opConfig) as Validator; } function encode(str: string, type: string) { diff --git a/packages/ts-transforms/tsconfig.json b/packages/ts-transforms/tsconfig.json index d0c6849850f..aae2752e13c 100644 --- a/packages/ts-transforms/tsconfig.json +++ b/packages/ts-transforms/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": "." + "rootDir": ".", + "suppressImplicitAnyIndexErrors": false }, "include": ["src", "test"] } From 94cca5f772fd43a75c85465cea5907fe7ce8633c Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 20 Dec 2024 09:39:30 -0700 Subject: [PATCH 04/21] fix type errors in teraslice-state-storage --- .../src/cached-state-storage/index.ts | 2 +- .../test/elasticsearch-state-storage-spec.ts | 59 ++++++++++++++++--- .../teraslice-state-storage/tsconfig.json | 3 +- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/packages/teraslice-state-storage/src/cached-state-storage/index.ts b/packages/teraslice-state-storage/src/cached-state-storage/index.ts index c7665fe48a2..821298f33ec 100644 --- a/packages/teraslice-state-storage/src/cached-state-storage/index.ts +++ b/packages/teraslice-state-storage/src/cached-state-storage/index.ts @@ -18,7 +18,7 @@ export default class CachedStateStorage extends EventEmitter { } mget(keyArray: (string | number)[]): MGetCacheResponse { - return keyArray.reduce((cachedState, key) => { + return keyArray.reduce((cachedState: Record, key) => { const state = this.get(key); if (state) cachedState[key] = state; return cachedState; diff --git a/packages/teraslice-state-storage/test/elasticsearch-state-storage-spec.ts b/packages/teraslice-state-storage/test/elasticsearch-state-storage-spec.ts index 855bb8e4815..c30c19c39ce 100644 --- a/packages/teraslice-state-storage/test/elasticsearch-state-storage-spec.ts +++ b/packages/teraslice-state-storage/test/elasticsearch-state-storage-spec.ts @@ -1,8 +1,13 @@ import 'jest-extended'; import { DataEntity, debugLogger, times } from '@terascope/utils'; -import { ClientParams, ClientResponse } from '@terascope/types'; +import { + BulkIndexOperation, BulkOperationContainer, + BulkUpdateAction, ClientParams, ClientResponse +} from '@terascope/types'; import { ESCachedStateStorage, ESStateStorageConfig } from '../src/index.js'; +type BulkOperationContainerWithIndex = BulkOperationContainer & { index: BulkIndexOperation }; + // TODO: this should search against elasticsearch describe('elasticsearch-state-storage', () => { const logger = debugLogger('elasticsearch-state-storage'); @@ -300,11 +305,20 @@ describe('elasticsearch-state-storage', () => { await stateStorage.mset(docArray); expect(client._bulkRequest).toBeArrayOfSize(6); - expect(client._bulkRequest[0].index._id).toBe('key-0'); + expect( + isBulkOperationContainerWithIndex(client._bulkRequest[0]) + && client._bulkRequest[0].index._id + ).toBe('key-0'); expect(client._bulkRequest[1]).toEqual(docArray[0]); - expect(client._bulkRequest[2].index._id).toBe('key-1'); + expect( + isBulkOperationContainerWithIndex(client._bulkRequest[2]) + && client._bulkRequest[2].index._id + ).toBe('key-1'); expect(client._bulkRequest[3]).toEqual(docArray[1]); - expect(client._bulkRequest[4].index._id).toBe('key-2'); + expect( + isBulkOperationContainerWithIndex(client._bulkRequest[4]) + && client._bulkRequest[4].index._id + ).toBe('key-2'); expect(client._bulkRequest[5]).toEqual(docArray[2]); }); }); @@ -352,11 +366,20 @@ describe('elasticsearch-state-storage', () => { await stateStorage.mset(otherDocArray); expect(client._bulkRequest).toBeArrayOfSize(20); - expect(client._bulkRequest[0].index._id).toBe('other-0'); + expect( + isBulkOperationContainerWithIndex(client._bulkRequest[0]) + && client._bulkRequest[0].index._id + ).toBe('other-0'); expect(client._bulkRequest[1]).toEqual(otherDocArray[0]); - expect(client._bulkRequest[2].index._id).toBe('other-1'); + expect( + isBulkOperationContainerWithIndex(client._bulkRequest[2]) + && client._bulkRequest[2].index._id + ).toBe('other-1'); expect(client._bulkRequest[3]).toEqual(otherDocArray[1]); - expect(client._bulkRequest[4].index._id).toBe('other-2'); + expect( + isBulkOperationContainerWithIndex(client._bulkRequest[4]) + && client._bulkRequest[4].index._id + ).toBe('other-2'); expect(client._bulkRequest[5]).toEqual(otherDocArray[2]); }); }); @@ -499,10 +522,26 @@ function copyDataEntity(doc: DataEntity): DataEntity { return DataEntity.make(updated, doc.getMetadata()); } +function isBulkOperationContainerWithIndex(obj: any): obj is BulkOperationContainerWithIndex { + return ( + typeof obj === 'object' + && obj !== null + && 'index' in obj + && typeof obj.index._index === 'string' + && typeof obj.index._type === 'string' + && typeof obj.index._id === 'string' + ); +} + class TestClient { private _getResponse!: ClientResponse.GetResponse; private _mgetResponse!: ClientResponse.MGetResponse; - _bulkRequest!: ClientParams.BulkParams>; + _bulkRequest!: ( + Record + | BulkOperationContainer + | BulkUpdateAction, unknown> + )[]; + private _config: ESStateStorageConfig; constructor(config: ESStateStorageConfig) { @@ -599,7 +638,9 @@ class TestClient { } async bulk(request: ClientParams.BulkParams>) { - this._bulkRequest = request.body as any; + if (request.body) { + this._bulkRequest = request.body; + } let i = -1; return { errors: false, diff --git a/packages/teraslice-state-storage/tsconfig.json b/packages/teraslice-state-storage/tsconfig.json index d0c6849850f..aae2752e13c 100644 --- a/packages/teraslice-state-storage/tsconfig.json +++ b/packages/teraslice-state-storage/tsconfig.json @@ -2,7 +2,8 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": "." + "rootDir": ".", + "suppressImplicitAnyIndexErrors": false }, "include": ["src", "test"] } From 445cb364e6a375592cb860a98fe4d17341f6517a Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 14:20:04 -0700 Subject: [PATCH 05/21] remove all suppressImplicitAnyIndexErrors rules --- packages/data-mate/tsconfig.json | 3 +-- packages/elasticsearch-store/tsconfig.json | 3 +-- packages/job-components/tsconfig.json | 3 +-- packages/scripts/tsconfig.json | 3 +-- packages/terafoundation/tsconfig.json | 3 +-- packages/teraslice-cli/tsconfig.json | 3 +-- packages/teraslice-client-js/tsconfig.json | 3 +-- packages/teraslice-messaging/tsconfig.json | 3 +-- packages/teraslice-state-storage/tsconfig.json | 3 +-- packages/ts-transforms/tsconfig.json | 3 +-- packages/utils/tsconfig.json | 3 +-- packages/xlucene-parser/tsconfig.json | 3 +-- tsconfig.json | 1 - 13 files changed, 12 insertions(+), 25 deletions(-) diff --git a/packages/data-mate/tsconfig.json b/packages/data-mate/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/data-mate/tsconfig.json +++ b/packages/data-mate/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/elasticsearch-store/tsconfig.json b/packages/elasticsearch-store/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/elasticsearch-store/tsconfig.json +++ b/packages/elasticsearch-store/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/job-components/tsconfig.json b/packages/job-components/tsconfig.json index 41cdbf3aa53..011671a87b4 100644 --- a/packages/job-components/tsconfig.json +++ b/packages/job-components/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false, + "rootDir": "." }, "include": [ "src", diff --git a/packages/scripts/tsconfig.json b/packages/scripts/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/scripts/tsconfig.json +++ b/packages/scripts/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/terafoundation/tsconfig.json b/packages/terafoundation/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/terafoundation/tsconfig.json +++ b/packages/terafoundation/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/teraslice-cli/tsconfig.json b/packages/teraslice-cli/tsconfig.json index 6305da286bc..3fc6692159f 100644 --- a/packages/teraslice-cli/tsconfig.json +++ b/packages/teraslice-cli/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": [ "src", diff --git a/packages/teraslice-client-js/tsconfig.json b/packages/teraslice-client-js/tsconfig.json index 9b1b1c82537..234151afc50 100644 --- a/packages/teraslice-client-js/tsconfig.json +++ b/packages/teraslice-client-js/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": [ "src", diff --git a/packages/teraslice-messaging/tsconfig.json b/packages/teraslice-messaging/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/teraslice-messaging/tsconfig.json +++ b/packages/teraslice-messaging/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/teraslice-state-storage/tsconfig.json b/packages/teraslice-state-storage/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/teraslice-state-storage/tsconfig.json +++ b/packages/teraslice-state-storage/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/ts-transforms/tsconfig.json b/packages/ts-transforms/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/ts-transforms/tsconfig.json +++ b/packages/ts-transforms/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/utils/tsconfig.json b/packages/utils/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/utils/tsconfig.json +++ b/packages/utils/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/packages/xlucene-parser/tsconfig.json b/packages/xlucene-parser/tsconfig.json index aae2752e13c..d0c6849850f 100644 --- a/packages/xlucene-parser/tsconfig.json +++ b/packages/xlucene-parser/tsconfig.json @@ -2,8 +2,7 @@ "extends": "../../tsconfig", "compilerOptions": { "outDir": "dist", - "rootDir": ".", - "suppressImplicitAnyIndexErrors": false + "rootDir": "." }, "include": ["src", "test"] } diff --git a/tsconfig.json b/tsconfig.json index 68227f86ce9..26426434598 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,6 @@ "esModuleInterop": true, "resolveJsonModule": true, "forceConsistentCasingInFileNames": true, - "suppressImplicitAnyIndexErrors": true, "ignoreDeprecations": "5.0", "composite": true, "declaration": true, From 76bc4ca2ff44bf2061d6b6f24474364e9f8305e4 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 14:21:41 -0700 Subject: [PATCH 06/21] update typescript from 5.2.2 to 5.7.2 --- package.json | 6 +++--- packages/eslint-config/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 246fe0b0b37..562c008fb8b 100644 --- a/package.json +++ b/package.json @@ -67,10 +67,10 @@ "jest-extended": "~4.0.2", "jest-watch-typeahead": "~2.2.2", "node-notifier": "~10.0.1", - "patch-package": "^8.0.0", - "postinstall-postinstall": "^2.1.0", + "patch-package": "~8.0.0", + "postinstall-postinstall": "~2.1.0", "ts-jest": "~29.2.5", - "typescript": "~5.2.2" + "typescript": "~5.7.2" }, "engines": { "node": ">=18.18.0", diff --git a/packages/eslint-config/package.json b/packages/eslint-config/package.json index a279bb246f8..29bdb020c20 100644 --- a/packages/eslint-config/package.json +++ b/packages/eslint-config/package.json @@ -33,7 +33,7 @@ "eslint-plugin-react-hooks": "~5.1.0", "eslint-plugin-testing-library": "~7.1.1", "globals": "~15.13.0", - "typescript": "~5.2.2", + "typescript": "~5.7.2", "typescript-eslint": "~8.18.0" }, "engines": { From 948924df777e268a298b31af85fd1268e2e4f1da Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 14:21:59 -0700 Subject: [PATCH 07/21] add types for gc-stats --- packages/teraslice/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 05469511293..257dafe7399 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -52,7 +52,7 @@ "event-loop-stats": "~1.4.1", "express": "~4.21.2", "fs-extra": "~11.2.0", - "gc-stats": "~1.4.0", + "gc-stats": "1.4.1", "get-port": "~7.1.0", "got": "~13.0.0", "ip": "~2.0.1", @@ -68,6 +68,7 @@ "devDependencies": { "@types/archiver": "~6.0.2", "@types/express": "~4.17.21", + "@types/gc-stats": "~1.4.3", "archiver": "~7.0.1", "bufferstreams": "~3.0.0", "chance": "~1.1.12", From 07b81279bdb8dd898d461095256beb4105d6ae7f Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 14:22:11 -0700 Subject: [PATCH 08/21] update yarn.lock --- yarn.lock | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/yarn.lock b/yarn.lock index 1660b76db9b..291a48acfde 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2648,6 +2648,13 @@ "@types/jsonfile" "*" "@types/node" "*" +"@types/gc-stats@^1.4.3": + version "1.4.3" + resolved "https://registry.yarnpkg.com/@types/gc-stats/-/gc-stats-1.4.3.tgz#2dc35998fdb6032870816b92b704ee12899f1523" + integrity sha512-Aek/WiRt7EHDKFNI3hDjYVvPNEeSjWGxN/8jxyaFoAawAq/bd8geI04XPMLQlLBO7LneLoWAAAvFv7hedgdi3w== + dependencies: + "@types/node" "*" + "@types/geojson@^7946.0.10", "@types/geojson@^7946.0.14", "@types/geojson@~7946.0.15": version "7946.0.15" resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.15.tgz#f9d55fd5a0aa2de9dc80b1b04e437538b7298868" @@ -5529,7 +5536,7 @@ functions-have-names@^1.2.3: resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== -gc-stats@~1.4.0: +gc-stats@1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/gc-stats/-/gc-stats-1.4.1.tgz#c9142dc54d02af9e93472b0bd972e6bacecc7be5" integrity sha512-eAvDBpI6UjVIYwLxshPCJJIkPyfamIrJzBtW/103+ooJWkISS+chVnHNnsZ+ubaw2607rFeiRDNWHkNUA+ioqg== @@ -8117,7 +8124,7 @@ parseurl@~1.3.3: resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== -patch-package@^8.0.0: +patch-package@~8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/patch-package/-/patch-package-8.0.0.tgz#d191e2f1b6e06a4624a0116bcb88edd6714ede61" integrity sha512-da8BVIhzjtgScwDJ2TtKsfT5JFWz1hYoBl9rUQ1f38MC2HwnEIkK8VN3dKMKcP7P7bvvgzNDbfNHtx3MsQb5vA== @@ -8298,7 +8305,7 @@ possible-typed-array-names@^1.0.0: resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q== -postinstall-postinstall@^2.1.0: +postinstall-postinstall@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/postinstall-postinstall/-/postinstall-postinstall-2.1.0.tgz#4f7f77441ef539d1512c40bd04c71b06a4704ca3" integrity sha512-7hQX6ZlZXIoRiWNrbMQaLzUUfH+sSx39u8EJ9HYuDc1kLo9IXKWjM5RSquZN1ad5GnH8CGFM78fsAAQi3OKEEQ== @@ -9784,10 +9791,10 @@ typescript-eslint@~8.18.0: "@typescript-eslint/parser" "8.18.0" "@typescript-eslint/utils" "8.18.0" -typescript@~5.2.2: - version "5.2.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" - integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== +typescript@~5.7.2: + version "5.7.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.7.2.tgz#3169cf8c4c8a828cde53ba9ecb3d2b1d5dd67be6" + integrity sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg== typpy@^2.3.1: version "2.3.13" From 1a2b872bde4be14d9d6c73c8fe894a6480218956 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 15:11:32 -0700 Subject: [PATCH 09/21] fix type errors in teraslice package --- .../teraslice/src/lib/cluster/node_master.ts | 2 +- .../teraslice/src/lib/cluster/services/api.ts | 49 ++++++++++++--- .../cluster/backends/kubernetesV2/k8sState.ts | 5 +- .../services/cluster/backends/native/index.ts | 8 +-- .../cluster/backends/native/messaging.ts | 60 +++++++++++++------ .../storage/backends/elasticsearch_store.ts | 2 +- packages/teraslice/src/lib/storage/state.ts | 8 +-- packages/teraslice/src/lib/utils/api_utils.ts | 12 ++-- .../teraslice/src/lib/utils/date_utils.ts | 4 +- .../execution-analytics.ts | 21 +++++-- .../lib/workers/execution-controller/index.ts | 12 ++-- .../src/lib/workers/metrics/index.ts | 9 +-- .../kubernetes/k8sState-multicluster-spec.ts | 5 +- .../backends/kubernetes/k8sState-spec.ts | 14 ++++- .../v2/k8sState-multicluster-v2-spec.ts | 5 +- .../kubernetes/v2/k8sState-v2-spec.ts | 14 ++++- packages/teraslice/test/node_master-spec.ts | 2 +- .../teraslice/test/services/messaging-spec.ts | 6 +- .../teraslice/test/utils/asset_utils-spec.ts | 2 +- .../test/workers/helpers/test-context.ts | 2 +- .../test/workers/worker/slice-spec.ts | 17 +++--- packages/ts-transforms/src/loader/utils.ts | 2 +- packages/types/src/teraslice.ts | 10 ++-- packages/xlucene-parser/src/parser.ts | 2 +- 24 files changed, 182 insertions(+), 91 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/node_master.ts b/packages/teraslice/src/lib/cluster/node_master.ts index 54380a8e9a7..e147b4c6904 100644 --- a/packages/teraslice/src/lib/cluster/node_master.ts +++ b/packages/teraslice/src/lib/cluster/node_master.ts @@ -399,7 +399,7 @@ export async function nodeMaster(context: ClusterMasterContext) { query: { node_id: context.sysconfig._nodeName } - } as any); + }); if (context.sysconfig.teraslice.master) { logger.debug(`node ${context.sysconfig._nodeName} is creating the cluster_master`); diff --git a/packages/teraslice/src/lib/cluster/services/api.ts b/packages/teraslice/src/lib/cluster/services/api.ts index aeb1b7af8be..cdd2c6c3b34 100644 --- a/packages/teraslice/src/lib/cluster/services/api.ts +++ b/packages/teraslice/src/lib/cluster/services/api.ts @@ -1,10 +1,12 @@ import { Router, Express } from 'express'; +import type { ParsedQs } from 'qs'; import bodyParser from 'body-parser'; import { pipeline as streamPipeline } from 'node:stream/promises'; import { RecoveryCleanupType, TerasliceConfig } from '@terascope/job-components'; import { parseErrorInfo, parseList, logError, - TSError, startsWith, Logger, pWhile + TSError, startsWith, Logger, pWhile, + isKey } from '@terascope/utils'; import { ExecutionStatusEnum } from '@terascope/types'; import { ClusterMasterContext, TerasliceRequest, TerasliceResponse } from '../../../interfaces.js'; @@ -17,7 +19,7 @@ import { createJobActiveQuery, addDeletedToQuery } from '../../utils/api_utils.js'; import { getPackageJSON } from '../../utils/file_utils.js'; -import got from 'got'; +import got, { OptionsInit } from 'got'; const terasliceVersion = getPackageJSON().version; @@ -78,7 +80,7 @@ export class ApiService { } queryKeys.forEach((key) => { - if (keyOptions[key]) { + if (key in keyOptions) { msg = key; workerNum = Number(query[key]); } @@ -135,24 +137,51 @@ export class ApiService { throw error; } + private _parsedQsToSearchParams(parsedQs: ParsedQs) { + const searchParams: Record = {}; + + for (const [key, value] of Object.entries(parsedQs)) { + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + searchParams[key] = value; + } else if (Array.isArray(value)) { + searchParams[key] = value.join(','); + } else if (value === null || value === undefined) { + // Skip undefined or null values + } else { + // stringify objects + searchParams[key] = JSON.stringify(value); + } + } + + return searchParams; + } + private async _redirect(req: TerasliceRequest, res: TerasliceResponse) { - const options = { + const searchParams = this._parsedQsToSearchParams(req.query); + + const options: OptionsInit & { isStream: true } = { prefixUrl: this.assetsUrl, headers: req.headers, - searchParams: req.query, + searchParams, throwHttpErrors: false, timeout: { request: this.terasliceConfig.api_response_timeout }, decompress: false, - retry: { limit: 0 } + retry: { limit: 0 }, + isStream: true }; const uri = req.url.replace(/^\//, ''); const method = req.method.toLowerCase(); try { + if (!isKey(got.stream, method)) { + throw new Error(`${method} is not a valid gotStream method`); + } + const stream = got.stream[method](uri, options); + await streamPipeline( req, - got.stream[method](uri, options), + stream, res, ); } catch (err) { @@ -852,7 +881,9 @@ export class ApiService { ex.workers ); for (const status in ExecutionStatusEnum) { - if (ExecutionStatusEnum[status]) { + if ( + isKey(ExecutionStatusEnum, status) + ) { const statusLabels = { ...controllerLabels, status: ExecutionStatusEnum[status] @@ -875,7 +906,7 @@ export class ApiService { const clusterState = this.clusterService.getClusterState(); /// Filter out information about kubernetes ex pods - const filteredExecutions = {}; + const filteredExecutions: Record = {}; for (const node in clusterState) { if (clusterState[node].active) { for (const worker of clusterState[node].active) { diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts index 6008430837c..0e6bbf10def 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts @@ -2,6 +2,7 @@ import { get, has, uniq, difference } from '@terascope/utils'; import { TSPodList } from './interfaces'; +import { ClusterState, ProcessAssignment } from '@terascope/types'; /** * Given the k8s Pods API output generates the appropriate Teraslice cluster @@ -12,7 +13,7 @@ import { TSPodList } from './interfaces'; * @param {String} clusterNameLabel k8s label containing clusterName * @param {Logger} logger Teraslice logger */ -export function gen(k8sPods: TSPodList, clusterState: Record) { +export function gen(k8sPods: TSPodList, clusterState: ClusterState) { // Make sure we clean up the old const hostIPs = uniq(k8sPods.items.map((item) => get(item, 'status.hostIP'))); const oldHostIps = difference(Object.keys(clusterState), hostIPs); @@ -46,7 +47,7 @@ export function gen(k8sPods: TSPodList, clusterState: Record) { const worker = { assets: [], - assignment: pod.metadata.labels['app.kubernetes.io/component'], + assignment: pod.metadata.labels['app.kubernetes.io/component'] as ProcessAssignment, ex_id: pod.metadata.labels['teraslice.terascope.io/exId'], // WARNING: This makes the assumption that the first container // in the pod is the teraslice container. Currently it is the diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts index 224f5cb1751..9308477843f 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts @@ -192,7 +192,7 @@ export class NativeClustering { throw new Error('Missing required stores'); } const server = this.clusterMasterServer.httpServer; - // @ts-expect-error + await this.messaging.listen({ server }); this.clusterStateInterval = setInterval(() => { @@ -447,7 +447,7 @@ export class NativeClustering { async allocateSlicer(ex: ExecutionConfig): Promise { let retryCount = 0; const errorNodes = {}; - // @ts-expect-error + const _allocateSlicer = async () => { try { return await this._createSlicer(ex, errorNodes); @@ -499,8 +499,8 @@ export class NativeClustering { const workers = findWorkersByExecutionID(this.clusterState, exId); let workerCount = workerNum; - const workersData = workers.reduce((prev, curr) => { - if (!prev[curr.node_id]) { + const workersData = workers.reduce((prev: Record, curr) => { + if (prev[curr.node_id]) { prev[curr.node_id] = 1; } else { prev[curr.node_id] += 1; diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts index a0e9dd7f481..6702544fc6d 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts @@ -1,9 +1,12 @@ /* eslint-disable prefer-const */ import type { EventEmitter } from 'node:events'; +import type { Server as HttpServer } from 'node:http'; +import type { Server as HttpsServer } from 'node:https'; + import { nanoid } from 'nanoid'; import { pDelay, Queue, Logger, isFunction, - isEmpty, get, toNumber + isEmpty, get, toNumber, isKey } from '@terascope/utils'; import { Context } from '@terascope/job-components'; import socketIOClient from 'socket.io-client'; @@ -75,6 +78,11 @@ export const routing = Object.freeze({ type HookFN = () => void | null; +type ListenOptions = { + server?: string | number | HttpServer | HttpsServer; + query?: { node_id: string }; +}; + export class Messaging { context: Context; logger: Logger; @@ -235,7 +243,7 @@ export class Messaging { private _registerFns(socket: any) { for (const [key, func] of Object.entries(this.functionMapping)) { if (func.__socketIdentifier) { - const wrappedFunc = (msg = {}) => { + const wrappedFunc = (msg: Record = {}) => { const identifier = func.__socketIdentifier; let id = msg[identifier]; // if already set, extract value else set it on socket @@ -272,11 +280,15 @@ export class Messaging { private _determinePathForMessage(messageSent: any) { const { to } = messageSent; - let destinationType = routing[this.self][to]; + let destinationType: string | undefined = undefined; + + if (isKey(routing, this.self) && isKey(routing[this.self], to)) { + destinationType = routing[this.self][to]; + } // cluster_master has two types of connections to node_master, if it does not have a // address then its talking to its own node_master through ipc // TODO: reference self message, remove cluster_master specific code - if (this.self === 'cluster_master' && !messageSent.address && clusterMasterMessages.ipc[messageSent.message]) { + if (this.self === 'cluster_master' && !messageSent.address && (messageSent.message in clusterMasterMessages.ipc)) { destinationType = 'ipc'; } if (destinationType === undefined) { @@ -304,8 +316,8 @@ export class Messaging { }); } - // @ts-expect-error - listen({ server, query } = {}) { + listen(options: ListenOptions = {}) { + const { query, server } = options; this.messsagingOnline = true; if (this.config.clients.networkClient) { @@ -313,9 +325,8 @@ export class Messaging { this.io = socketIOClient(this.hostURL, { forceNew: true, path: '/native-clustering', - perMessageDeflate: false, - query, - } as any); + query + }); this._registerFns(this.io); @@ -334,14 +345,25 @@ export class Messaging { this.logger.debug('client network connection is online'); } else if (server) { - // cluster_master - this.io = socketIOServer(server, { - path: '/native-clustering', - pingTimeout: this.configTimeout, - pingInterval: this.configTimeout + this.networkLatencyBuffer, - perMessageDeflate: false, - serveClient: false, - }); + if (typeof server === 'string' || typeof server === 'number') { + // test + this.io = socketIOServer(server, { + path: '/native-clustering', + pingTimeout: this.configTimeout, + pingInterval: this.configTimeout + this.networkLatencyBuffer, + perMessageDeflate: false, + serveClient: false, + }); + } else { + // cluster_master + this.io = socketIOServer(server, { + path: '/native-clustering', + pingTimeout: this.configTimeout, + pingInterval: this.configTimeout + this.networkLatencyBuffer, + perMessageDeflate: false, + serveClient: false, + }); + } this._attachRoomsSocketIO(); this.io.on('connection', (socket: any) => { @@ -439,7 +461,7 @@ export class Messaging { const processConfig: Record = {}; // @ts-expect-error const testProcess = this.context.__testingModule; - processConfig.clients = options[env.assignment]; + processConfig.clients = options[env.assignment as keyof typeof options]; if (processConfig.clients.ipcClient) { // all children of node_master @@ -529,7 +551,7 @@ export class Messaging { 'cluster:slicer:analytics': 'cluster:slicer:analytics', }; - return stateQuery[msg] !== undefined; + return msg in stateQuery; } private _emitIpcMessage(fn: any) { diff --git a/packages/teraslice/src/lib/storage/backends/elasticsearch_store.ts b/packages/teraslice/src/lib/storage/backends/elasticsearch_store.ts index 553095a52a1..9f5abe304f4 100644 --- a/packages/teraslice/src/lib/storage/backends/elasticsearch_store.ts +++ b/packages/teraslice/src/lib/storage/backends/elasticsearch_store.ts @@ -144,7 +144,7 @@ export class TerasliceElasticsearchStorage { if (isMultiIndex) { // @ts-expect-error TODO: fix this - const storeType = this.defaultIndexName.match(/__(.*)\*/)[1]; + const storeType: 'analytics' | 'state' = this.defaultIndexName.match(/__(.*)\*/)[1]; const timeseriesFormat = config.index_rollover_frequency[storeType]; const nameSize = this.defaultIndexName.length - 1; newIndex = timeseriesIndex( diff --git a/packages/teraslice/src/lib/storage/state.ts b/packages/teraslice/src/lib/storage/state.ts index 777094ee163..94610466996 100644 --- a/packages/teraslice/src/lib/storage/state.ts +++ b/packages/teraslice/src/lib/storage/state.ts @@ -2,7 +2,7 @@ import { Context, RecoveryCleanupType, Slice } from '@terascope/job-components'; import { TSError, pRetry, toString, isRetryableError, parseErrorInfo, isTest, - times, getFullErrorStack, Logger + times, getFullErrorStack, Logger, isKey } from '@terascope/utils'; import { timeseriesIndex, TimeseriesFormat } from '../utils/date_utils.js'; import { makeLogger } from '../workers/helpers/terafoundation.js'; @@ -82,7 +82,7 @@ export class StateStorage { } private _createSliceRecord(exId: string, slice: any, state: any, error?: Error) { - if (!SliceState[state]) { + if (!isKey(SliceState, state)) { throw new Error(`Unknown slice state "${state}" on create`); } const { index } = timeseriesIndex( @@ -109,7 +109,7 @@ export class StateStorage { // TODO: type this better async updateState(slice: Slice, state: string, error?: Error) { - if (!SliceState[state]) { + if (!isKey(SliceState, state)) { throw new Error(`Unknown slice state "${state}" on update`); } @@ -283,7 +283,7 @@ export class StateStorage { } async countByState(exId: string, state: string) { - if (!SliceState[state]) { + if (!isKey(SliceState, state)) { throw new Error(`Unknown slice state "${state}" on update`); } const query = `ex_id:"${exId}" AND state:${state}`; diff --git a/packages/teraslice/src/lib/utils/api_utils.ts b/packages/teraslice/src/lib/utils/api_utils.ts index c7f4adc7233..908d2b20009 100644 --- a/packages/teraslice/src/lib/utils/api_utils.ts +++ b/packages/teraslice/src/lib/utils/api_utils.ts @@ -1,9 +1,11 @@ import Table from 'easy-table'; import { parseErrorInfo, parseList, logError, - isString, get, toInteger, Logger, TSError, + isString, get, toInteger, Logger, + TSError, isKey, } from '@terascope/utils'; import { TerasliceRequest, TerasliceResponse } from '../../interfaces.js'; +import type { ClusterMaster } from '@terascope/teraslice-messaging'; export function makeTable( req: TerasliceRequest, @@ -103,7 +105,7 @@ export function sendError( // NOTE: This only works for counters, if you're trying to extend this, you // should probably switch to using prom-client. -export function makePrometheus(stats: any, defaultLabels = {}) { +export function makePrometheus(stats: ClusterMaster.ClusterAnalytics, defaultLabels = {}) { const metricMapping = { processed: 'teraslice_slices_processed', failed: 'teraslice_slices_failed', @@ -116,8 +118,8 @@ export function makePrometheus(stats: any, defaultLabels = {}) { let returnString = ''; Object.entries(stats.controllers).forEach(([key, value]) => { - const name = metricMapping[key]; - if (name !== '') { + if (isKey(metricMapping, key)) { + const name = metricMapping[key]; returnString += `# TYPE ${name} counter\n`; const labels = makePrometheusLabels(defaultLabels); returnString += `${name}${labels} ${value}\n`; @@ -132,7 +134,7 @@ function makePrometheusLabels(defaults = {}) { if (!keys.length) return ''; const labelsStr = keys.map((key) => { - const val = labels[key]; + const val = labels[key as keyof typeof labels]; return `${key}="${val}"`; }).join(','); diff --git a/packages/teraslice/src/lib/utils/date_utils.ts b/packages/teraslice/src/lib/utils/date_utils.ts index 13fe2d3d494..3d524e021f2 100644 --- a/packages/teraslice/src/lib/utils/date_utils.ts +++ b/packages/teraslice/src/lib/utils/date_utils.ts @@ -1,4 +1,4 @@ -import { makeISODate } from '@terascope/utils'; +import { isKey, makeISODate } from '@terascope/utils'; const options = { year: 'y', @@ -44,7 +44,7 @@ const formatter = { export type TimeseriesFormat = 'daily' | 'monthly' | 'yearly'; export function dateOptions(value: string | undefined) { - if (value && options[value]) { + if (value && isKey(options, value)) { return options[value]; } diff --git a/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts b/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts index 30d876c3e98..1cca469810d 100644 --- a/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts +++ b/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts @@ -1,6 +1,6 @@ import { makeISODate, get, has, - Logger + Logger, isKey } from '@terascope/utils'; import type { EventEmitter } from 'node:events'; import type { Context, ExecutionContext } from '@terascope/job-components'; @@ -133,11 +133,17 @@ export class ExecutionAnalytics { this.executionAnalytics[key] += 1; } - get(key: string) { - if (key) { + get(key: 'started' | 'queuing_complete'): Date | string | number | undefined; + get(key: undefined): EStats; + get(key: keyof Omit): number; + get(key: keyof EStats | undefined): EStats | Date | string | number | undefined { + if (key === 'started' || key === 'queuing_complete') { return this.executionAnalytics[key]; } - return this.executionAnalytics; + if (key === undefined) { + return this.executionAnalytics; + } + return this.executionAnalytics[key]; } getAnalytics() { @@ -169,8 +175,11 @@ export class ExecutionAnalytics { const copy: Partial = {}; Object.entries(this.pushedAnalytics).forEach(([field, value]) => { - diffs[field] = analytics[field] - value; - copy[field] = analytics[field]; + // if field is a key of copy, it is also a key of diffs and analytics + if (isKey(copy, field)) { + diffs[field] = analytics[field] - value; + copy[field] = analytics[field]; + } }); const response = await this.client.sendClusterAnalytics( diff --git a/packages/teraslice/src/lib/workers/execution-controller/index.ts b/packages/teraslice/src/lib/workers/execution-controller/index.ts index 657448226b4..ffefcffa4ec 100644 --- a/packages/teraslice/src/lib/workers/execution-controller/index.ts +++ b/packages/teraslice/src/lib/workers/execution-controller/index.ts @@ -43,7 +43,7 @@ export class ExecutionController { isExecutionDone = false; private workersHaveConnected = false; - private _handlers = new Map void>(); + private _handlers = new Map void) | null>(); executionAnalytics: ExecutionAnalytics; readonly scheduler: Scheduler; private metrics: Metrics | null; @@ -297,7 +297,9 @@ export class ExecutionController { }); for (const [event, handler] of this._handlers.entries()) { - this.events.on(event, handler); + if (handler !== null) { + this.events.on(event, handler); + } } if (this.collectAnalytics) { @@ -474,8 +476,10 @@ export class ExecutionController { // remove any listeners for (const [event, handler] of this._handlers.entries()) { - this.events.removeListener(event, handler); - this._handlers[event] = null; + if (handler !== null) { + this.events.removeListener(event, handler); + this._handlers.set(event, null); + } } this.isShuttingDown = true; diff --git a/packages/teraslice/src/lib/workers/metrics/index.ts b/packages/teraslice/src/lib/workers/metrics/index.ts index d808ad6a303..a4d0dcb1de5 100644 --- a/packages/teraslice/src/lib/workers/metrics/index.ts +++ b/packages/teraslice/src/lib/workers/metrics/index.ts @@ -2,6 +2,7 @@ import { EventEmitter } from 'node:events'; import { debugLogger, isTest, Logger } from '@terascope/utils'; +import GCStats from 'gc-stats'; const defaultLogger = debugLogger('metrics'); @@ -11,7 +12,7 @@ export class Metrics extends EventEmitter { private eventLoopInterval: number; // TODO: fix types here private _typesCollectedAt: Record; - private gcStats: any; + private gcStats: GCStats.GCStatsEventEmitter | null; private eventLoopStats: any; constructor(config?: { logger: Logger }) { @@ -26,12 +27,12 @@ export class Metrics extends EventEmitter { this.eventLoopInterval = isTest ? 100 : 5000; this._intervals = []; this._typesCollectedAt = {}; + this.gcStats = null; } async initialize() { // never cause an unwanted error try { - // @ts-expect-error const module = await import('gc-stats'); this.gcStats = module.default(); } catch (err) { @@ -52,7 +53,7 @@ export class Metrics extends EventEmitter { loopEnabled }); - if (gcEnabled) { + if (this.gcStats !== null) { // https://github.com/dainis/node-gcstats#property-insights const typesToName = { 1: 'Scavenge', @@ -62,7 +63,7 @@ export class Metrics extends EventEmitter { 15: 'All' }; - this.gcStats.on('stats', (metrics: any) => { + this.gcStats.on('stats', (metrics: GCStats.GCStatistics) => { // never cause an unwanted error if (!metrics) { this.logger.warn('invalid metrics received for gc stats', metrics); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-multicluster-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-multicluster-spec.ts index 07680287511..0df24208609 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-multicluster-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-multicluster-spec.ts @@ -1,11 +1,12 @@ import { cloneDeep } from '@terascope/utils'; +import { ClusterState } from '@terascope/types'; import _podsJobRunning from './files/job-running-v1-k8s-pods-multicluster.json'; import { gen } from '../../../../../../../src/lib/cluster/services/cluster/backends/kubernetes/k8sState.js'; describe('k8sState with pods from multiple clusters', () => { it('should generate cluster state correctly on first call', () => { const podsJobRunning = cloneDeep(_podsJobRunning) as any; - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); @@ -39,7 +40,7 @@ describe('k8sState with pods from multiple clusters', () => { it('should generate cluster state correctly on second call', () => { const podsJobRunning = cloneDeep(_podsJobRunning) as any; - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); gen(podsJobRunning, clusterState); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-spec.ts index f823b4904c2..3a8328b852d 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sState-spec.ts @@ -1,11 +1,12 @@ import { cloneDeep } from '@terascope/utils'; +import { ClusterState } from '@terascope/types'; import _podsJobRunning from './files/job-running-v1-k8s-pods.json'; import { gen } from '../../../../../../../src/lib/cluster/services/cluster/backends/kubernetes/k8sState.js'; describe('k8sState', () => { it('should generate cluster state correctly on first call', () => { const podsJobRunning = cloneDeep(_podsJobRunning) as any; - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); @@ -38,7 +39,7 @@ describe('k8sState', () => { it('should generate cluster state correctly on second call', () => { const podsJobRunning = cloneDeep(_podsJobRunning) as any; - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); gen(podsJobRunning, clusterState); @@ -71,9 +72,16 @@ describe('k8sState', () => { it('should remove old host ips', () => { const podsJobRunning = cloneDeep(_podsJobRunning) as any; - const clusterState = {}; + const clusterState: ClusterState = {}; clusterState['2.2.2.2'] = { + node_id: '2.2.2.2', + hostname: '2.2.2.2', + pid: 'N/A', + node_version: 'N/A', + teraslice_version: 'N/A', + total: 'N/A', state: 'idk', + available: 'N/A', active: [] }; diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts index 8de3af9bb91..870de4806e8 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-multicluster-v2-spec.ts @@ -1,4 +1,5 @@ import { cloneDeep } from '@terascope/utils'; +import { ClusterState } from '@terascope/types'; import _podsJobRunning from '../files/job-running-v1-k8s-pods-multicluster.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; import { TSPodList } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; @@ -6,7 +7,7 @@ import { TSPodList } from '../../../../../../../../src/lib/cluster/services/clus describe('k8sState with pods from multiple clusters', () => { it('should generate cluster state correctly on first call', () => { const podsJobRunning = cloneDeep(_podsJobRunning as any); - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); @@ -40,7 +41,7 @@ describe('k8sState with pods from multiple clusters', () => { it('should generate cluster state correctly on second call', () => { const podsJobRunning = cloneDeep(_podsJobRunning as any); - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); gen(podsJobRunning, clusterState); diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts index 35235068ea4..34e100df0c8 100644 --- a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts +++ b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/v2/k8sState-v2-spec.ts @@ -1,4 +1,5 @@ import { cloneDeep } from '@terascope/utils'; +import { ClusterState } from '@terascope/types'; import _podsJobRunning from '../files/job-running-v1-k8s-pods.json'; import { gen } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.js'; import { TSPodList } from '../../../../../../../../src/lib/cluster/services/cluster/backends/kubernetesV2/interfaces.js'; @@ -6,7 +7,7 @@ import { TSPodList } from '../../../../../../../../src/lib/cluster/services/clus describe('k8sState', () => { it('should generate cluster state correctly on first call', () => { const podsJobRunning = cloneDeep(_podsJobRunning as any); - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); // console.log(`clusterState\n\n${JSON.stringify(clusterState, null, 2)}`); @@ -39,7 +40,7 @@ describe('k8sState', () => { it('should generate cluster state correctly on second call', () => { const podsJobRunning = cloneDeep(_podsJobRunning as any); - const clusterState = {}; + const clusterState: ClusterState = {}; gen(podsJobRunning, clusterState); gen(podsJobRunning, clusterState); @@ -72,9 +73,16 @@ describe('k8sState', () => { it('should remove old host ips', () => { const podsJobRunning = cloneDeep(_podsJobRunning as any); - const clusterState = {}; + const clusterState: ClusterState = {}; clusterState['2.2.2.2'] = { + node_id: '2.2.2.2', + hostname: '2.2.2.2', + pid: 'N/A', + node_version: 'N/A', + teraslice_version: 'N/A', + total: 'N/A', state: 'idk', + available: 'N/A', active: [] }; diff --git a/packages/teraslice/test/node_master-spec.ts b/packages/teraslice/test/node_master-spec.ts index 444b1fba5f9..c5640599587 100644 --- a/packages/teraslice/test/node_master-spec.ts +++ b/packages/teraslice/test/node_master-spec.ts @@ -97,7 +97,7 @@ describe('Node master', () => { processCounter += 1; for (const [key, value] of Object.entries(envConfig)) { - this[key] = value; + this[key as keyof this] = value; } this.process = { diff --git a/packages/teraslice/test/services/messaging-spec.ts b/packages/teraslice/test/services/messaging-spec.ts index 2e6f2aabf69..252f3774da4 100644 --- a/packages/teraslice/test/services/messaging-spec.ts +++ b/packages/teraslice/test/services/messaging-spec.ts @@ -5,7 +5,7 @@ import { Messaging, routing } from '../../src/lib/cluster/services/cluster/backe describe('messaging module', () => { const logger = debugLogger('messaging'); - let connected = {}; + let connected: Record }> = {}; const testExId = '7890'; @@ -15,7 +15,7 @@ describe('messaging module', () => { let clusterFn: any = () => {}; function testMessaging(messaging: Messaging, fn: string) { - return (...args: any[]) => messaging[fn](...args); + return (...args: any[]) => messaging[fn as keyof Messaging](...args); } class MyCluster extends events.EventEmitter { @@ -494,7 +494,7 @@ describe('messaging module', () => { const messaging2 = new Messaging(testContext2, logger); expect(() => messaging1.listen()).not.toThrow(); - expect(() => messaging2.listen({ server: 45645 } as any)).not.toThrow(); + expect(() => messaging2.listen({ server: 45645 })).not.toThrow(); await messaging1.shutdown(); await messaging2.shutdown(); diff --git a/packages/teraslice/test/utils/asset_utils-spec.ts b/packages/teraslice/test/utils/asset_utils-spec.ts index 6c1d2724053..05a26f9c6f4 100644 --- a/packages/teraslice/test/utils/asset_utils-spec.ts +++ b/packages/teraslice/test/utils/asset_utils-spec.ts @@ -49,7 +49,7 @@ describe('Asset Utils', () => { }); }); - const wrongPlatform = { + const wrongPlatform: Record = { darwin: 'linux', linux: 'darwin', }; diff --git a/packages/teraslice/test/workers/helpers/test-context.ts b/packages/teraslice/test/workers/helpers/test-context.ts index f5e77b09a64..8d4f2605148 100644 --- a/packages/teraslice/test/workers/helpers/test-context.ts +++ b/packages/teraslice/test/workers/helpers/test-context.ts @@ -55,7 +55,7 @@ export class TestContext { assignment: string; clusterMaster!: ClusterMaster.Server; _stores: TestStoreContainer = {}; - cleanups = {}; + cleanups: Record Promise> = {}; constructor(options: TestContextArgs = {}) { const { diff --git a/packages/teraslice/test/workers/worker/slice-spec.ts b/packages/teraslice/test/workers/worker/slice-spec.ts index 73927e4cf4d..3815af99bdf 100644 --- a/packages/teraslice/test/workers/worker/slice-spec.ts +++ b/packages/teraslice/test/workers/worker/slice-spec.ts @@ -4,7 +4,10 @@ import { SliceExecution } from '../../../src/lib/workers/worker/slice.js'; import { TestContext } from '../helpers/index.js'; describe('Slice', () => { - async function setupSlice(testContext: any, eventMocks = {}): Promise { + async function setupSlice( + testContext: any, + eventMocks: Record = {} + ): Promise { await testContext.initialize(); await testContext.executionContext.initialize(); @@ -44,7 +47,7 @@ describe('Slice', () => { let slice: SliceExecution; let results: any; let testContext: any; - const eventMocks = {}; + const eventMocks: Record = {}; beforeEach(async () => { testContext = new TestContext({ analytics: true }); @@ -93,7 +96,7 @@ describe('Slice', () => { let slice: SliceExecution; let results: any; let testContext: any; - const eventMocks = {}; + const eventMocks: Record = {}; beforeEach(async () => { testContext = new TestContext({ analytics: false }); @@ -136,7 +139,7 @@ describe('Slice', () => { let slice: SliceExecution; let results: any; let testContext: any; - const eventMocks = {}; + const eventMocks: Record = {}; beforeEach(async () => { testContext = new TestContext({ @@ -184,7 +187,7 @@ describe('Slice', () => { let slice: SliceExecution; let err: Error; let testContext: any; - const eventMocks = {}; + const eventMocks: Record = {}; beforeEach(async () => { testContext = new TestContext({ @@ -238,7 +241,7 @@ describe('Slice', () => { describe('when the slice fails', () => { let slice: SliceExecution; let testContext: any; - const eventMocks = {}; + const eventMocks: Record = {}; let err: Error; beforeEach(async () => { @@ -290,7 +293,7 @@ describe('Slice', () => { describe('when the slice fails with zero retries', () => { let slice: SliceExecution; let testContext: any; - const eventMocks = {}; + const eventMocks: Record = {}; let err: Error; beforeEach(async () => { diff --git a/packages/ts-transforms/src/loader/utils.ts b/packages/ts-transforms/src/loader/utils.ts index 16b4566d0c0..0fc3ee98fa5 100644 --- a/packages/ts-transforms/src/loader/utils.ts +++ b/packages/ts-transforms/src/loader/utils.ts @@ -298,7 +298,7 @@ export function isSimplePostProcessConfig(config: Config) { return !has(config, 'follow') && hasPostProcess(config); } -export function hasExtractions(config: Config) { +export function hasExtractions(config: Config): boolean { return has(config, 'source') || has(config, 'exp'); } diff --git a/packages/types/src/teraslice.ts b/packages/types/src/teraslice.ts index 014da5ef3a8..855f9fe7d47 100644 --- a/packages/types/src/teraslice.ts +++ b/packages/types/src/teraslice.ts @@ -523,8 +523,8 @@ export interface SysConfig extends BaseSysconfig {} export type Assignment = 'assets_service' | 'cluster_master' | 'node_master' | 'execution_controller' | 'worker'; interface BaseWorkerNode { - worker_id: number; - pid: number; + worker_id: string | number; + pid?: number; } export enum ProcessAssignment { @@ -572,12 +572,12 @@ export type ProcessNode = ClusterNode export interface NodeState { node_id: string; hostname: string; - pid: number; + pid: number | 'N/A'; node_version: string; teraslice_version: string; - total: number; + total: number | 'N/A'; state: string; - available: number; + available: number | 'N/A'; active: ProcessNode[]; } diff --git a/packages/xlucene-parser/src/parser.ts b/packages/xlucene-parser/src/parser.ts index cf6d4c137a1..815b20e4fc9 100644 --- a/packages/xlucene-parser/src/parser.ts +++ b/packages/xlucene-parser/src/parser.ts @@ -7,7 +7,7 @@ import { parse } from './peg-engine.js'; import * as i from './interfaces.js'; import * as utils from './utils.js'; -const termTypes = new Set(utils.termTypes.filter((type) => ( +const termTypes = new Set(utils.termTypes.filter((type) => ( type !== i.NodeType.Range && type !== i.NodeType.Function ))); From fb986ae2141208ff736081b90d262ab2e3a1831c Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 15:14:02 -0700 Subject: [PATCH 10/21] fix yarn.lock and resolutions --- packages/scripts/package.json | 4 ++-- yarn.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 8c3e4a5e0f9..67648a907d0 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -29,7 +29,7 @@ }, "resolutions": { "ms": "~2.1.3", - "typescript": "~5.2.2" + "typescript": "~5.7.2" }, "dependencies": { "@kubernetes/client-node": "~0.22.3", @@ -64,7 +64,7 @@ "@types/toposort": "~2.0.7" }, "peerDependencies": { - "typescript": "~5.2.2" + "typescript": "~5.7.2" }, "peerDependenciesMeta": { "typescript": { diff --git a/yarn.lock b/yarn.lock index 291a48acfde..8efad45e8bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2648,7 +2648,7 @@ "@types/jsonfile" "*" "@types/node" "*" -"@types/gc-stats@^1.4.3": +"@types/gc-stats@~1.4.3": version "1.4.3" resolved "https://registry.yarnpkg.com/@types/gc-stats/-/gc-stats-1.4.3.tgz#2dc35998fdb6032870816b92b704ee12899f1523" integrity sha512-Aek/WiRt7EHDKFNI3hDjYVvPNEeSjWGxN/8jxyaFoAawAq/bd8geI04XPMLQlLBO7LneLoWAAAvFv7hedgdi3w== From de6006c8c2d56fcf98c4039ff316dc751675cde9 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 15:46:54 -0700 Subject: [PATCH 11/21] fix makePrometheus() function --- packages/teraslice/src/lib/utils/api_utils.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/teraslice/src/lib/utils/api_utils.ts b/packages/teraslice/src/lib/utils/api_utils.ts index 908d2b20009..11dcf22d387 100644 --- a/packages/teraslice/src/lib/utils/api_utils.ts +++ b/packages/teraslice/src/lib/utils/api_utils.ts @@ -117,12 +117,15 @@ export function makePrometheus(stats: ClusterMaster.ClusterAnalytics, defaultLab }; let returnString = ''; + Object.entries(stats.controllers).forEach(([key, value]) => { if (isKey(metricMapping, key)) { const name = metricMapping[key]; - returnString += `# TYPE ${name} counter\n`; - const labels = makePrometheusLabels(defaultLabels); - returnString += `${name}${labels} ${value}\n`; + if (name !== '') { + returnString += `# TYPE ${name} counter\n`; + const labels = makePrometheusLabels(defaultLabels); + returnString += `${name}${labels} ${value}\n`; + } } }); return returnString; From 13cd68cd74eed7edb97bd8632ff646aa1ffc8d94 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 2 Jan 2025 16:46:34 -0700 Subject: [PATCH 12/21] fix _pushAnalytics() --- .../execution-controller/execution-analytics.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts b/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts index 1cca469810d..82c4e0e96e9 100644 --- a/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts +++ b/packages/teraslice/src/lib/workers/execution-controller/execution-analytics.ts @@ -1,6 +1,6 @@ import { makeISODate, get, has, - Logger, isKey + Logger } from '@terascope/utils'; import type { EventEmitter } from 'node:events'; import type { Context, ExecutionContext } from '@terascope/job-components'; @@ -171,15 +171,12 @@ export class ExecutionAnalytics { const analytics = this.getAnalytics(); // save a copy of what we push so we can emit diffs - const diffs: Partial = {}; - const copy: Partial = {}; + const diffs: Record = {}; + const copy: Record = {}; Object.entries(this.pushedAnalytics).forEach(([field, value]) => { - // if field is a key of copy, it is also a key of diffs and analytics - if (isKey(copy, field)) { - diffs[field] = analytics[field] - value; - copy[field] = analytics[field]; - } + diffs[field] = analytics[field as keyof Omit] - value; + copy[field] = analytics[field as keyof EStats]; }); const response = await this.client.sendClusterAnalytics( From 0a8e56dd102c90a633cb272a8858b13f1b40af07 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 3 Jan 2025 08:12:46 -0700 Subject: [PATCH 13/21] fix typo --- .../src/lib/cluster/services/cluster/backends/native/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts index 9308477843f..0550988a90a 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/index.ts @@ -500,7 +500,7 @@ export class NativeClustering { let workerCount = workerNum; const workersData = workers.reduce((prev: Record, curr) => { - if (prev[curr.node_id]) { + if (!prev[curr.node_id]) { prev[curr.node_id] = 1; } else { prev[curr.node_id] += 1; From abb3f55b8226c6e9059591e75ec620f543824c90 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 3 Jan 2025 15:52:25 -0700 Subject: [PATCH 14/21] refactor --- packages/data-mate/src/index.ts | 4 ++-- packages/elasticsearch-store/src/index-model.ts | 2 +- packages/elasticsearch-store/src/utils/model.ts | 4 ++-- .../test/index-manager-index-setup-spec.ts | 2 +- packages/teraslice/src/interfaces.ts | 13 +++++++++++++ .../teraslice/src/lib/cluster/services/api.ts | 7 +++---- .../cluster/backends/kubernetesV2/index.ts | 3 ++- .../services/cluster/backends/native/messaging.ts | 15 ++++++++++----- packages/teraslice/src/lib/utils/api_utils.ts | 1 - .../teraslice/src/lib/workers/metrics/index.ts | 2 +- packages/ts-transforms/src/command.ts | 10 +++++++--- packages/ts-transforms/src/operations/index.ts | 1 + .../src/operations/lib/validations/base.ts | 4 ++-- .../src/operations/plugins/validator/index.ts | 14 ++++---------- packages/types/src/xlucene-interfaces.ts | 4 ++++ 15 files changed, 53 insertions(+), 33 deletions(-) diff --git a/packages/data-mate/src/index.ts b/packages/data-mate/src/index.ts index 51a12e69c04..37d8ab7c146 100644 --- a/packages/data-mate/src/index.ts +++ b/packages/data-mate/src/index.ts @@ -29,7 +29,7 @@ declare module './aggregation-frame/AggregationFrame.d.ts' { * @returns the new columns */ run(): Promise>; - [key: string]: any; // fixMe: make sure this is okay + [key: string | symbol]: unknown; } } @@ -58,7 +58,7 @@ AggregationFrame.prototype.run = async function run() { const aggregationFrameMethods = Reflect.ownKeys(AggregationFrame.prototype); for (const dataFrameMethod of Reflect.ownKeys(DataFrame.prototype)) { if (!aggregationFrameMethods.includes(dataFrameMethod)) { - AggregationFrame.prototype[dataFrameMethod as keyof AggregationFrame] = () => { + AggregationFrame.prototype[dataFrameMethod] = () => { throw new Error( `Unsupported method on ${String(dataFrameMethod)} AggregationFrame. Use it before DataFrame.aggregate or after AggregationFrame.run()` diff --git a/packages/elasticsearch-store/src/index-model.ts b/packages/elasticsearch-store/src/index-model.ts index 33d373b2e82..66f64691b47 100644 --- a/packages/elasticsearch-store/src/index-model.ts +++ b/packages/elasticsearch-store/src/index-model.ts @@ -227,7 +227,7 @@ export abstract class IndexModel extends IndexStor const entries = Object.entries(this._sanitizeFields); for (const [field, method] of entries) { - if (isKey(record, field)) { + if (isKey(record, field) && record[field]) { switch (method) { case 'trim': this._setStringValueOrThrow( diff --git a/packages/elasticsearch-store/src/utils/model.ts b/packages/elasticsearch-store/src/utils/model.ts index de15d59c0ae..71d054f3d0f 100644 --- a/packages/elasticsearch-store/src/utils/model.ts +++ b/packages/elasticsearch-store/src/utils/model.ts @@ -63,9 +63,9 @@ export function mergeDefaults(source: T, from: Partial): T const _mapping = from ? cloneDeep(from) : {}; for (const [key, val] of Object.entries(_mapping)) { - if (isKey(output, key)) { + if (isKey(output, key) && output[key] != null) { if (isPlainObject(val)) { - output[key] = Object.assign(output[key] as object, val) as T[string & keyof T]; + output[key] = Object.assign(output[key], val); } else if (Array.isArray(val)) { output[key] = concat(output[key], val) as T[string & keyof T]; } else { diff --git a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts index 670151a5f8d..7247dc4228c 100644 --- a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts +++ b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts @@ -6,7 +6,7 @@ import { IndexManager, timeSeriesIndex, IndexConfig, getESVersion, __timeSeriesTest, ElasticsearchTestHelpers } from '../src/index.js'; -import { MappingTypeMapping } from 'opensearch1/api/types.js'; +import { MappingTypeMapping } from '@terascope/types'; const { makeClient, cleanupIndex, TEST_INDEX_PREFIX, diff --git a/packages/teraslice/src/interfaces.ts b/packages/teraslice/src/interfaces.ts index f52c8dd10bb..840e52ac2eb 100644 --- a/packages/teraslice/src/interfaces.ts +++ b/packages/teraslice/src/interfaces.ts @@ -29,12 +29,18 @@ export interface ClusterMasterContext extends Context { } export enum ProcessAssignment { + node_master = 'node_master', cluster_master = 'cluster_master', assets_service = 'assets_service', execution_controller = 'execution_controller', worker = 'worker' } + +export function isProcessAssignment(value: string): value is ProcessAssignment { + return Object.values(ProcessAssignment).includes(value as ProcessAssignment); +} + interface BaseWorkerNode { worker_id: number; pid: number; @@ -99,3 +105,10 @@ export interface ControllerStats extends ExecutionAnalytics { job_id: string; name: string; } + +export type MessagingConfigOptions = { + [assignment in ProcessAssignment]: { + networkClient: boolean; + ipcClient: boolean; + }; +}; diff --git a/packages/teraslice/src/lib/cluster/services/api.ts b/packages/teraslice/src/lib/cluster/services/api.ts index cdd2c6c3b34..ae4a0afe2ec 100644 --- a/packages/teraslice/src/lib/cluster/services/api.ts +++ b/packages/teraslice/src/lib/cluster/services/api.ts @@ -1,5 +1,4 @@ import { Router, Express } from 'express'; -import type { ParsedQs } from 'qs'; import bodyParser from 'body-parser'; import { pipeline as streamPipeline } from 'node:stream/promises'; import { RecoveryCleanupType, TerasliceConfig } from '@terascope/job-components'; @@ -137,10 +136,10 @@ export class ApiService { throw error; } - private _parsedQsToSearchParams(parsedQs: ParsedQs) { + private queryToSearchParams(query: any) { const searchParams: Record = {}; - for (const [key, value] of Object.entries(parsedQs)) { + for (const [key, value] of Object.entries(query)) { if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { searchParams[key] = value; } else if (Array.isArray(value)) { @@ -157,7 +156,7 @@ export class ApiService { } private async _redirect(req: TerasliceRequest, res: TerasliceResponse) { - const searchParams = this._parsedQsToSearchParams(req.query); + const searchParams = this.queryToSearchParams(req.query); const options: OptionsInit & { isStream: true } = { prefixUrl: this.assetsUrl, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 1246eff21e3..4a07eb978a2 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -12,6 +12,7 @@ import { ResourceType } from './interfaces.js'; import { K8sJobResource } from './k8sJobResource.js'; import { K8sServiceResource } from './k8sServiceResource.js'; import { K8sDeploymentResource } from './k8sDeploymentResource.js'; +import { ClusterState } from '@terascope/types'; /* Execution Life Cycle for _status @@ -27,7 +28,7 @@ export class KubernetesClusterBackendV2 { k8s: K8s; logger: Logger; private clusterStateInterval: NodeJS.Timeout | undefined; - clusterState: Record = {}; + clusterState: ClusterState = {}; readonly clusterNameLabel: string; constructor(context: Context, clusterMasterServer: any) { diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts index 6702544fc6d..c22cbce06a2 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts @@ -11,6 +11,7 @@ import { import { Context } from '@terascope/job-components'; import socketIOClient from 'socket.io-client'; import socketIOServer from 'socket.io'; +import { isProcessAssignment, MessagingConfigOptions, ProcessAssignment } from '../../../../../../interfaces.js'; // messages send to cluster_master const clusterMasterMessages = { @@ -449,7 +450,7 @@ export class Messaging { private _makeConfigurations() { let host; let port; - const options = { + const options: MessagingConfigOptions = { node_master: { networkClient: true, ipcClient: false }, cluster_master: { networkClient: false, ipcClient: true }, execution_controller: { networkClient: false, ipcClient: true }, @@ -461,7 +462,11 @@ export class Messaging { const processConfig: Record = {}; // @ts-expect-error const testProcess = this.context.__testingModule; - processConfig.clients = options[env.assignment as keyof typeof options]; + const { assignment } = env; + if (!isProcessAssignment(assignment)) { + throw new Error(`assignment must be on of: ${Object.values(ProcessAssignment).toString()}. Received ${assignment}`); + } + processConfig.clients = options[assignment]; if (processConfig.clients.ipcClient) { // all children of node_master @@ -472,14 +477,14 @@ export class Messaging { } if (processConfig.clients.networkClient) { - if (env.assignment === 'node_master' || env.assignment === 'assets_service') { + if (assignment === 'node_master' || assignment === 'assets_service') { host = this.context.sysconfig.teraslice.master_hostname; ({ port } = this.context.sysconfig.teraslice); } processConfig.hostURL = this._makeHostName(host as string, port as unknown as string); } - processConfig.assignment = env.assignment; + processConfig.assignment = assignment; return processConfig; } @@ -551,7 +556,7 @@ export class Messaging { 'cluster:slicer:analytics': 'cluster:slicer:analytics', }; - return msg in stateQuery; + return (isKey(stateQuery, msg) && stateQuery[msg] !== undefined); } private _emitIpcMessage(fn: any) { diff --git a/packages/teraslice/src/lib/utils/api_utils.ts b/packages/teraslice/src/lib/utils/api_utils.ts index 11dcf22d387..af20f9f6de4 100644 --- a/packages/teraslice/src/lib/utils/api_utils.ts +++ b/packages/teraslice/src/lib/utils/api_utils.ts @@ -117,7 +117,6 @@ export function makePrometheus(stats: ClusterMaster.ClusterAnalytics, defaultLab }; let returnString = ''; - Object.entries(stats.controllers).forEach(([key, value]) => { if (isKey(metricMapping, key)) { const name = metricMapping[key]; diff --git a/packages/teraslice/src/lib/workers/metrics/index.ts b/packages/teraslice/src/lib/workers/metrics/index.ts index a4d0dcb1de5..3e28335233e 100644 --- a/packages/teraslice/src/lib/workers/metrics/index.ts +++ b/packages/teraslice/src/lib/workers/metrics/index.ts @@ -2,7 +2,7 @@ import { EventEmitter } from 'node:events'; import { debugLogger, isTest, Logger } from '@terascope/utils'; -import GCStats from 'gc-stats'; +import type GCStats from 'gc-stats'; const defaultLogger = debugLogger('metrics'); diff --git a/packages/ts-transforms/src/command.ts b/packages/ts-transforms/src/command.ts index cc49b173b76..ac11294501b 100644 --- a/packages/ts-transforms/src/command.ts +++ b/packages/ts-transforms/src/command.ts @@ -8,9 +8,9 @@ import { DataEntity, debugLogger, parseList, AnyObject, get, pMap } from '@terascope/utils'; +import { isXLuceneFieldType, xLuceneTypeConfig } from '@terascope/types'; import { PhaseManager } from './index.js'; import { PhaseConfig } from './interfaces.js'; -import { xLuceneFieldType } from 'packages/types/dist/src/xlucene-interfaces.js'; const dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -50,7 +50,7 @@ const filePath = command.rules as string; const dataPath = command.data as string; const streamData = command.f && command.f === 'ldjson' ? command.f : false; const ignoreErrors = command.i || false; -let typesConfig: Record = {}; // fixme: confirm it's always one of these +let typesConfig: xLuceneTypeConfig = {}; const type = command.m ? 'matcher' : 'transform'; interface ESData { @@ -65,7 +65,11 @@ try { if (pieces.length !== 2) { throw new Error(`Expected -t option line #${index} to have key:value pair format, got ${segment}`); } - typesConfig[pieces[0].trim()] = pieces[1].trim() as xLuceneFieldType; + const fieldType = pieces[1].trim(); + if (!isXLuceneFieldType(fieldType)) { + throw new Error(`Expected -t option line #${index} value of ${fieldType} to be of type xLuceneFieldType`); + } + typesConfig[pieces[0].trim()] = fieldType; }); } if (command.T) { diff --git a/packages/ts-transforms/src/operations/index.ts b/packages/ts-transforms/src/operations/index.ts index c459b447a01..012058969a2 100644 --- a/packages/ts-transforms/src/operations/index.ts +++ b/packages/ts-transforms/src/operations/index.ts @@ -80,6 +80,7 @@ class OperationsManager { constructor(pluginList: PluginList = []) { pluginList.push(CorePlugins); + // @ts-expect-error FIXME: try to remove this ignore pluginList.push(ValidatorPlugins); pluginList.push(dataMatePlugin); diff --git a/packages/ts-transforms/src/operations/lib/validations/base.ts b/packages/ts-transforms/src/operations/lib/validations/base.ts index 253f965b7bd..2759f692914 100644 --- a/packages/ts-transforms/src/operations/lib/validations/base.ts +++ b/packages/ts-transforms/src/operations/lib/validations/base.ts @@ -2,11 +2,11 @@ import { DataEntity, get, isFunction, isNil } from '@terascope/utils'; import OperationBase from '../base.js'; +import { OperationConfig } from '../../../interfaces.js'; export default abstract class ValidationOpBase extends OperationBase { private invert: boolean; - // @ts-expect-error - constructor(config) { + constructor(config: OperationConfig) { super(config); this.invert = this.config.output === false; } diff --git a/packages/ts-transforms/src/operations/plugins/validator/index.ts b/packages/ts-transforms/src/operations/plugins/validator/index.ts index bf0340b947b..12a0d11f9d6 100644 --- a/packages/ts-transforms/src/operations/plugins/validator/index.ts +++ b/packages/ts-transforms/src/operations/plugins/validator/index.ts @@ -1,7 +1,7 @@ import { deprecate } from 'util'; import validator from 'validator'; import ValidationOpBase from '../../lib/validations/base.js'; -import { PostProcessConfig, PluginClassType, InputOutputCardinality, OperationsDict } from '../../../interfaces.js'; +import { PostProcessConfig, PluginClassType, InputOutputCardinality } from '../../../interfaces.js'; export class Validator extends ValidationOpBase { private method: keyof typeof validator; @@ -24,14 +24,7 @@ export class Validator extends ValidationOpBase { } } -interface ValidatorInterfaceType { - new (config: PostProcessConfig): Validator; - cardinality: InputOutputCardinality; - -} - -function setup(method: keyof typeof validator): ValidatorInterfaceType { - // @ts-expect-error fixMe: properly type a class returning a different class +function setup(method: keyof typeof validator) { return class ValidatorInterface { static cardinality: InputOutputCardinality = 'one-to-one'; @@ -42,7 +35,8 @@ function setup(method: keyof typeof validator): ValidatorInterfaceType { } export class ValidatorPlugins implements PluginClassType { - init(): OperationsDict { + // @ts-expect-error + init() { return { after: deprecate(setup('isAfter'), 'after is being deprecated., please use isAfter instead', 'after'), alpha: deprecate(setup('isAlpha'), 'alpha is being deprecated, please use isAlpha instead', 'alpha'), diff --git a/packages/types/src/xlucene-interfaces.ts b/packages/types/src/xlucene-interfaces.ts index f1c5cd0358f..9c44256db48 100644 --- a/packages/types/src/xlucene-interfaces.ts +++ b/packages/types/src/xlucene-interfaces.ts @@ -14,6 +14,10 @@ export enum xLuceneFieldType { Number = 'number', } +export function isXLuceneFieldType(value: string): value is xLuceneFieldType { + return Object.values(xLuceneFieldType).includes(value as xLuceneFieldType); +} + export interface xLuceneTypeConfig { [field: string]: xLuceneFieldType; } From c7a0440f48a277d5279d45bafc00fed44bb6c5c0 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 3 Jan 2025 16:20:05 -0700 Subject: [PATCH 15/21] better comment --- .../lib/cluster/services/cluster/backends/native/messaging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts index c22cbce06a2..e56859f1121 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts @@ -347,7 +347,7 @@ export class Messaging { this.logger.debug('client network connection is online'); } else if (server) { if (typeof server === 'string' || typeof server === 'number') { - // test + // test server this.io = socketIOServer(server, { path: '/native-clustering', pingTimeout: this.configTimeout, From 859bfd3ab9c9e628531a0b4310b08f01036c74c9 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 7 Jan 2025 08:32:54 -0700 Subject: [PATCH 16/21] fix import order --- .../elasticsearch-store/test/index-manager-index-setup-spec.ts | 2 +- .../lib/cluster/services/cluster/backends/kubernetesV2/index.ts | 2 +- .../cluster/services/cluster/backends/kubernetesV2/k8sState.ts | 2 +- packages/teraslice/src/lib/utils/api_utils.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts index 7247dc4228c..4df9f8640a0 100644 --- a/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts +++ b/packages/elasticsearch-store/test/index-manager-index-setup-spec.ts @@ -1,12 +1,12 @@ import 'jest-extended'; import { debugLogger, get, isKey } from '@terascope/utils'; +import { MappingTypeMapping } from '@terascope/types'; import * as simple from './helpers/simple-index.js'; import * as template from './helpers/template-index.js'; import { IndexManager, timeSeriesIndex, IndexConfig, getESVersion, __timeSeriesTest, ElasticsearchTestHelpers } from '../src/index.js'; -import { MappingTypeMapping } from '@terascope/types'; const { makeClient, cleanupIndex, TEST_INDEX_PREFIX, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts index 4a07eb978a2..a2311e5564c 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/index.ts @@ -3,6 +3,7 @@ import { cloneDeep, pRetry, Logger } from '@terascope/utils'; import type { Context, ExecutionConfig } from '@terascope/job-components'; +import { ClusterState } from '@terascope/types'; import { makeLogger } from '../../../../../workers/helpers/terafoundation.js'; import { gen } from './k8sState.js'; import { K8s } from './k8s.js'; @@ -12,7 +13,6 @@ import { ResourceType } from './interfaces.js'; import { K8sJobResource } from './k8sJobResource.js'; import { K8sServiceResource } from './k8sServiceResource.js'; import { K8sDeploymentResource } from './k8sDeploymentResource.js'; -import { ClusterState } from '@terascope/types'; /* Execution Life Cycle for _status diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts index 0e6bbf10def..83bf18f5b2a 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/kubernetesV2/k8sState.ts @@ -1,8 +1,8 @@ import { get, has, uniq, difference } from '@terascope/utils'; -import { TSPodList } from './interfaces'; import { ClusterState, ProcessAssignment } from '@terascope/types'; +import { TSPodList } from './interfaces'; /** * Given the k8s Pods API output generates the appropriate Teraslice cluster diff --git a/packages/teraslice/src/lib/utils/api_utils.ts b/packages/teraslice/src/lib/utils/api_utils.ts index af20f9f6de4..fdfe738784a 100644 --- a/packages/teraslice/src/lib/utils/api_utils.ts +++ b/packages/teraslice/src/lib/utils/api_utils.ts @@ -4,8 +4,8 @@ import { isString, get, toInteger, Logger, TSError, isKey, } from '@terascope/utils'; -import { TerasliceRequest, TerasliceResponse } from '../../interfaces.js'; import type { ClusterMaster } from '@terascope/teraslice-messaging'; +import { TerasliceRequest, TerasliceResponse } from '../../interfaces.js'; export function makeTable( req: TerasliceRequest, From d59b53e37adde1f2a6e011c407dbf64a0385c190 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 7 Jan 2025 08:33:30 -0700 Subject: [PATCH 17/21] refactor type guards --- packages/teraslice/src/interfaces.ts | 2 +- packages/types/src/xlucene-interfaces.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/teraslice/src/interfaces.ts b/packages/teraslice/src/interfaces.ts index 840e52ac2eb..c429867b4e1 100644 --- a/packages/teraslice/src/interfaces.ts +++ b/packages/teraslice/src/interfaces.ts @@ -38,7 +38,7 @@ export enum ProcessAssignment { } export function isProcessAssignment(value: string): value is ProcessAssignment { - return Object.values(ProcessAssignment).includes(value as ProcessAssignment); + return value in ProcessAssignment; } interface BaseWorkerNode { diff --git a/packages/types/src/xlucene-interfaces.ts b/packages/types/src/xlucene-interfaces.ts index 9c44256db48..834e78517a3 100644 --- a/packages/types/src/xlucene-interfaces.ts +++ b/packages/types/src/xlucene-interfaces.ts @@ -14,8 +14,9 @@ export enum xLuceneFieldType { Number = 'number', } -export function isXLuceneFieldType(value: string): value is xLuceneFieldType { - return Object.values(xLuceneFieldType).includes(value as xLuceneFieldType); +export function isXLuceneFieldType(value: any): value is xLuceneFieldType { + const possibleValues = Object.values(xLuceneFieldType); + return possibleValues.includes(value); } export interface xLuceneTypeConfig { From f5044fb94a30f24682672c7c10bacc17d37f47c8 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 7 Jan 2025 08:34:45 -0700 Subject: [PATCH 18/21] refactor --- .../teraslice/src/lib/cluster/services/api.ts | 39 +++-------- .../cluster/backends/native/messaging.ts | 64 ++++++++++--------- .../teraslice/test/services/messaging-spec.ts | 2 +- 3 files changed, 43 insertions(+), 62 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/api.ts b/packages/teraslice/src/lib/cluster/services/api.ts index ae4a0afe2ec..4f950399098 100644 --- a/packages/teraslice/src/lib/cluster/services/api.ts +++ b/packages/teraslice/src/lib/cluster/services/api.ts @@ -1,6 +1,7 @@ import { Router, Express } from 'express'; import bodyParser from 'body-parser'; import { pipeline as streamPipeline } from 'node:stream/promises'; +import got, { OptionsInit } from 'got'; import { RecoveryCleanupType, TerasliceConfig } from '@terascope/job-components'; import { parseErrorInfo, parseList, logError, @@ -18,7 +19,6 @@ import { createJobActiveQuery, addDeletedToQuery } from '../../utils/api_utils.js'; import { getPackageJSON } from '../../utils/file_utils.js'; -import got, { OptionsInit } from 'got'; const terasliceVersion = getPackageJSON().version; @@ -136,32 +136,11 @@ export class ApiService { throw error; } - private queryToSearchParams(query: any) { - const searchParams: Record = {}; - - for (const [key, value] of Object.entries(query)) { - if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { - searchParams[key] = value; - } else if (Array.isArray(value)) { - searchParams[key] = value.join(','); - } else if (value === null || value === undefined) { - // Skip undefined or null values - } else { - // stringify objects - searchParams[key] = JSON.stringify(value); - } - } - - return searchParams; - } - - private async _redirect(req: TerasliceRequest, res: TerasliceResponse) { - const searchParams = this.queryToSearchParams(req.query); - - const options: OptionsInit & { isStream: true } = { + private async _assetRedirect(req: TerasliceRequest, res: TerasliceResponse) { + const options: OptionsInit & { isStream: true } = { prefixUrl: this.assetsUrl, headers: req.headers, - searchParams, + searchParams: req.query as Record, throwHttpErrors: false, timeout: { request: this.terasliceConfig.api_response_timeout }, decompress: false, @@ -223,7 +202,7 @@ export class ApiService { this.jobsStorage = jobsStorage; const v1routes = Router(); - const redirect = this._redirect.bind(this); + const assetRedirect = this._assetRedirect.bind(this); this.app.use(bodyParser.json({ type(req) { @@ -271,17 +250,17 @@ export class ApiService { v1routes.route('/assets*') .delete((req, res) => { - redirect(req as TerasliceRequest, res); + assetRedirect(req as TerasliceRequest, res); }) .post((req, res) => { if (req.headers['content-type'] === 'application/json' || req.headers['content-type'] === 'application/x-www-form-urlencoded') { sendError(res, 400, '/asset endpoints do not accept json'); return; } - redirect(req as TerasliceRequest, res); + assetRedirect(req as TerasliceRequest, res); }) // @ts-expect-error - .get(redirect); + .get(assetRedirect); v1routes.post('/jobs', (req, res) => { // if no job was posted an empty object is returned, so we check if it has values @@ -534,7 +513,7 @@ export class ApiService { this.app.route('/txt/assets*') // @ts-expect-error - .get(redirect); + .get(assetRedirect); this.app.get('/txt/workers', (req, res) => { const { size, from } = getSearchOptions(req as TerasliceRequest); diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts index e56859f1121..3ce1de54762 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts @@ -80,7 +80,8 @@ export const routing = Object.freeze({ type HookFN = () => void | null; type ListenOptions = { - server?: string | number | HttpServer | HttpsServer; + server?: HttpServer | HttpsServer; + port?: string | number; query?: { node_id: string }; }; @@ -317,10 +318,31 @@ export class Messaging { }); } + private _createIOServer(server?: HttpServer | HttpsServer, port?: string | number) { + const opts = { + path: '/native-clustering', + pingTimeout: this.configTimeout, + pingInterval: this.configTimeout + this.networkLatencyBuffer, + perMessageDeflate: false, + serveClient: false, + } + if (server) { + this.io = socketIOServer(server, opts); + } else if (port) { + this.io = socketIOServer(port, opts); + } + this._attachRoomsSocketIO(); + + this.io.on('connection', (socket: any) => { + this.logger.debug('a connection to cluster_master has been made'); + this._registerFns(socket); + }); + } + listen(options: ListenOptions = {}) { - const { query, server } = options; + const { query, server, port } = options; this.messsagingOnline = true; - + if (this.config.clients.networkClient) { // node_master, worker this.io = socketIOClient(this.hostURL, { @@ -328,9 +350,9 @@ export class Messaging { path: '/native-clustering', query }); - + this._registerFns(this.io); - + if (this.self === 'node_master') { this.io.on('networkMessage', (networkMsg: any) => { const { message } = networkMsg; @@ -343,34 +365,14 @@ export class Messaging { } }); } - + this.logger.debug('client network connection is online'); } else if (server) { - if (typeof server === 'string' || typeof server === 'number') { - // test server - this.io = socketIOServer(server, { - path: '/native-clustering', - pingTimeout: this.configTimeout, - pingInterval: this.configTimeout + this.networkLatencyBuffer, - perMessageDeflate: false, - serveClient: false, - }); - } else { - // cluster_master - this.io = socketIOServer(server, { - path: '/native-clustering', - pingTimeout: this.configTimeout, - pingInterval: this.configTimeout + this.networkLatencyBuffer, - perMessageDeflate: false, - serveClient: false, - }); - } - this._attachRoomsSocketIO(); - - this.io.on('connection', (socket: any) => { - this.logger.debug('a connection to cluster_master has been made'); - this._registerFns(socket); - }); + // cluster_master + this._createIOServer(server); + } else if (port) { + // test server + this._createIOServer(undefined, port); } // TODO: message queuing will be used until formal process lifecycles are implemented diff --git a/packages/teraslice/test/services/messaging-spec.ts b/packages/teraslice/test/services/messaging-spec.ts index 252f3774da4..b696933ce39 100644 --- a/packages/teraslice/test/services/messaging-spec.ts +++ b/packages/teraslice/test/services/messaging-spec.ts @@ -494,7 +494,7 @@ describe('messaging module', () => { const messaging2 = new Messaging(testContext2, logger); expect(() => messaging1.listen()).not.toThrow(); - expect(() => messaging2.listen({ server: 45645 })).not.toThrow(); + expect(() => messaging2.listen({ port: 45645 })).not.toThrow(); await messaging1.shutdown(); await messaging2.shutdown(); From 16c9a08c66447f88d7397ff6b8e16d63c4d03836 Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 7 Jan 2025 08:45:07 -0700 Subject: [PATCH 19/21] release: (minor) teraslice@2.11.0 bump: (minor) @terascope/data-mate@1.7.0, elasticsearch-store@1.7.0 bump: (minor) terafoundation@1.9.0, ts-transforms@1.7.0 bump: (minor) @terascope/teraslice-state-storage@1.7.0, @terascope/types@1.4.0 bump: (minor) @terascope/utils@1.7.0, @terascope/data-types@1.7.0 bump: (minor) xlucene-parser@1.7.0, xlucene-translator@1.7.0 bump: (minor) @terascope/elasticsearch-api@4.7.0, @terascope/job-components@1.9.0 --- e2e/package.json | 4 ++-- package.json | 2 +- packages/data-mate/package.json | 10 +++++----- packages/data-types/package.json | 6 +++--- packages/elasticsearch-api/package.json | 8 ++++---- packages/elasticsearch-store/package.json | 12 ++++++------ packages/job-components/package.json | 6 +++--- packages/scripts/package.json | 4 ++-- packages/terafoundation/package.json | 8 ++++---- packages/teraslice-cli/package.json | 6 +++--- packages/teraslice-client-js/package.json | 6 +++--- packages/teraslice-messaging/package.json | 6 +++--- packages/teraslice-state-storage/package.json | 6 +++--- packages/teraslice-test-harness/package.json | 4 ++-- packages/teraslice/package.json | 14 +++++++------- packages/ts-transforms/package.json | 8 ++++---- packages/types/package.json | 2 +- packages/utils/package.json | 4 ++-- packages/xlucene-parser/package.json | 6 +++--- packages/xlucene-translator/package.json | 8 ++++---- packages/xpressions/package.json | 6 +++--- 21 files changed, 68 insertions(+), 68 deletions(-) diff --git a/e2e/package.json b/e2e/package.json index 66e36629d24..c1df1746a08 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -43,9 +43,9 @@ "ms": "~2.1.3" }, "devDependencies": { - "@terascope/types": "~1.3.2", + "@terascope/types": "~1.4.0", "bunyan": "~1.8.15", - "elasticsearch-store": "~1.6.0", + "elasticsearch-store": "~1.7.0", "fs-extra": "~11.2.0", "ms": "~2.1.3", "nanoid": "~5.0.9", diff --git a/package.json b/package.json index 562c008fb8b..34761e7733c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "2.10.0", + "version": "2.11.0", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index 4f56e97673e..357bfc1fd19 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "1.6.0", + "version": "1.7.0", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { @@ -30,9 +30,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-types": "~1.6.0", - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/data-types": "~1.7.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "@types/validator": "~13.12.2", "awesome-phonenumber": "~7.2.0", "date-fns": "~4.1.0", @@ -45,7 +45,7 @@ "uuid": "~11.0.3", "valid-url": "~1.0.9", "validator": "~13.12.0", - "xlucene-parser": "~1.6.0" + "xlucene-parser": "~1.7.0" }, "devDependencies": { "@types/ip6addr": "~0.2.6", diff --git a/packages/data-types/package.json b/packages/data-types/package.json index ad26f2f9e2f..e542c72e4cf 100644 --- a/packages/data-types/package.json +++ b/packages/data-types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-types", "displayName": "Data Types", - "version": "1.6.0", + "version": "1.7.0", "description": "A library for defining the data structures and mapping", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme", "bugs": { @@ -27,8 +27,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "graphql": "~16.9.0", "yargs": "~17.7.2" }, diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index c784fe05cc8..61892db4c5a 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-api", "displayName": "Elasticsearch API", - "version": "4.6.0", + "version": "4.7.0", "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "bluebird": "~3.7.2", "setimmediate": "~1.0.5" }, @@ -33,7 +33,7 @@ "@opensearch-project/opensearch": "~1.2.0", "@types/elasticsearch": "~5.0.43", "elasticsearch": "~15.4.1", - "elasticsearch-store": "~1.6.0", + "elasticsearch-store": "~1.7.0", "elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0", "elasticsearch7": "npm:@elastic/elasticsearch@~7.17.0", "elasticsearch8": "npm:@elastic/elasticsearch@~8.15.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index 426e3655130..2d7cf257929 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "1.6.0", + "version": "1.7.0", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -30,10 +30,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "~1.6.0", - "@terascope/data-types": "~1.6.0", - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/data-mate": "~1.7.0", + "@terascope/data-types": "~1.7.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "ajv": "~8.17.1", "ajv-formats": "~3.0.1", "elasticsearch6": "npm:@elastic/elasticsearch@~6.8.0", @@ -43,7 +43,7 @@ "opensearch2": "npm:@opensearch-project/opensearch@~2.12.0", "setimmediate": "~1.0.5", "uuid": "~11.0.3", - "xlucene-translator": "~1.6.0" + "xlucene-translator": "~1.7.0" }, "devDependencies": { "@types/uuid": "~10.0.0" diff --git a/packages/job-components/package.json b/packages/job-components/package.json index 88b6aa03ddd..b4c8bc93a93 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "1.8.0", + "version": "1.9.0", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { @@ -32,8 +32,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "convict": "~6.2.4", "convict-format-with-moment": "~6.2.0", "convict-format-with-validator": "~6.2.0", diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 67648a907d0..ce81b46b6ed 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "1.7.1", + "version": "1.8.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -33,7 +33,7 @@ }, "dependencies": { "@kubernetes/client-node": "~0.22.3", - "@terascope/utils": "~1.6.0", + "@terascope/utils": "~1.7.0", "codecov": "~3.8.3", "execa": "~9.5.2", "fs-extra": "~11.2.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index 575182320a8..9b58cc34e1e 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "1.8.0", + "version": "1.9.0", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -29,15 +29,15 @@ }, "dependencies": { "@terascope/file-asset-apis": "~1.0.3", - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "bluebird": "~3.7.2", "bunyan": "~1.8.15", "convict": "~6.2.4", "convict-format-with-moment": "~6.2.0", "convict-format-with-validator": "~6.2.0", "elasticsearch": "~15.4.1", - "elasticsearch-store": "~1.6.0", + "elasticsearch-store": "~1.7.0", "express": "~4.21.2", "js-yaml": "~4.1.0", "nanoid": "~5.0.9", diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 5f3769c21fe..343bc9ef78a 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -43,8 +43,8 @@ }, "devDependencies": { "@terascope/fetch-github-release": "~1.0.0", - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "@types/decompress": "~4.2.7", "@types/diff": "~6.0.0", "@types/ejs": "~3.1.5", @@ -68,7 +68,7 @@ "pretty-bytes": "~6.1.1", "prompts": "~2.4.2", "signale": "~1.4.0", - "teraslice-client-js": "~1.6.0", + "teraslice-client-js": "~1.7.0", "tmp": "~0.2.0", "tty-table": "~4.2.3", "yargs": "~17.7.2" diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index f285d3d9f2e..0ece73d40ff 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "1.6.0", + "version": "1.7.0", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -32,8 +32,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "auto-bind": "~5.0.1", "got": "~13.0.0" }, diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index 4963755c1e8..e74d4388089 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-messaging", "displayName": "Teraslice Messaging", - "version": "1.9.0", + "version": "1.10.0", "description": "An internal teraslice messaging library using socket.io", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme", "bugs": { @@ -35,8 +35,8 @@ "ms": "~2.1.3" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "get-port": "~7.1.0", "ms": "~2.1.3", "nanoid": "~5.0.9", diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index 94642aa24c8..8ab068af6a3 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-state-storage", "displayName": "Teraslice State Storage", - "version": "1.6.0", + "version": "1.7.0", "description": "State storage operation api for teraslice", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme", "bugs": { @@ -24,8 +24,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/elasticsearch-api": "~4.6.0", - "@terascope/utils": "~1.6.0" + "@terascope/elasticsearch-api": "~4.7.0", + "@terascope/utils": "~1.7.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index ffccd1a1c51..36b009bf09e 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "~11.2.0" }, "devDependencies": { - "@terascope/job-components": "~1.8.0" + "@terascope/job-components": "~1.9.0" }, "peerDependencies": { - "@terascope/job-components": ">=1.8.0" + "@terascope/job-components": ">=1.9.0" }, "engines": { "node": ">=18.18.0", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 257dafe7399..924d6578e84 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "2.10.0", + "version": "2.11.0", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { @@ -39,11 +39,11 @@ }, "dependencies": { "@kubernetes/client-node": "~0.22.3", - "@terascope/elasticsearch-api": "~4.6.0", - "@terascope/job-components": "~1.8.0", - "@terascope/teraslice-messaging": "~1.9.0", - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/elasticsearch-api": "~4.7.0", + "@terascope/job-components": "~1.9.0", + "@terascope/teraslice-messaging": "~1.10.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "async-mutex": "~0.5.0", "barbe": "~3.0.16", "body-parser": "~1.20.2", @@ -62,7 +62,7 @@ "semver": "~7.6.3", "socket.io": "~1.7.4", "socket.io-client": "~1.7.4", - "terafoundation": "~1.8.0", + "terafoundation": "~1.9.0", "uuid": "~11.0.3" }, "devDependencies": { diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index 6d73cd70020..ac4b70894be 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "1.6.0", + "version": "1.7.0", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -36,9 +36,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "~1.6.0", - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/data-mate": "~1.7.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "awesome-phonenumber": "~7.2.0", "graphlib": "~2.1.8", "jexl": "~2.3.0", diff --git a/packages/types/package.json b/packages/types/package.json index cd374b2cb67..2c819a61773 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/types", "displayName": "Types", - "version": "1.3.2", + "version": "1.4.0", "description": "A collection of typescript interfaces", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/types#readme", "bugs": { diff --git a/packages/utils/package.json b/packages/utils/package.json index 95229c2a3bf..a186500cdfc 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/utils", "displayName": "Utils", - "version": "1.6.0", + "version": "1.7.0", "description": "A collection of Teraslice Utilities", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/utils#readme", "bugs": { @@ -30,7 +30,7 @@ }, "dependencies": { "@chainsafe/is-ip": "~2.0.2", - "@terascope/types": "~1.3.2", + "@terascope/types": "~1.4.0", "@turf/bbox": "~7.1.0", "@turf/bbox-polygon": "~7.1.0", "@turf/boolean-contains": "~7.1.0", diff --git a/packages/xlucene-parser/package.json b/packages/xlucene-parser/package.json index cd37b174bbc..fe54f401eab 100644 --- a/packages/xlucene-parser/package.json +++ b/packages/xlucene-parser/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-parser", "displayName": "xLucene Parser", - "version": "1.6.0", + "version": "1.7.0", "description": "Flexible Lucene-like evaluator and language parser", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-parser#readme", "repository": { @@ -33,8 +33,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "peggy": "~4.2.0", "ts-pegjs": "~4.2.1" }, diff --git a/packages/xlucene-translator/package.json b/packages/xlucene-translator/package.json index ef87343b2e9..1f56d36c317 100644 --- a/packages/xlucene-translator/package.json +++ b/packages/xlucene-translator/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-translator", "displayName": "xLucene Translator", - "version": "1.6.0", + "version": "1.7.0", "description": "Translate xlucene query to database queries", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-translator#readme", "repository": { @@ -29,10 +29,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "~1.3.2", - "@terascope/utils": "~1.6.0", + "@terascope/types": "~1.4.0", + "@terascope/utils": "~1.7.0", "@types/elasticsearch": "~5.0.43", - "xlucene-parser": "~1.6.0" + "xlucene-parser": "~1.7.0" }, "devDependencies": { "elasticsearch": "~15.4.1" diff --git a/packages/xpressions/package.json b/packages/xpressions/package.json index 9c448722a4f..ea60080aca9 100644 --- a/packages/xpressions/package.json +++ b/packages/xpressions/package.json @@ -1,7 +1,7 @@ { "name": "xpressions", "displayName": "Xpressions", - "version": "1.6.0", + "version": "1.7.0", "description": "Variable expressions with date-math support", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xpressions#readme", "bugs": { @@ -24,10 +24,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "~1.6.0" + "@terascope/utils": "~1.7.0" }, "devDependencies": { - "@terascope/types": "~1.3.2" + "@terascope/types": "~1.4.0" }, "engines": { "node": ">=18.18.0", From 14c703a798cda3349991c44707fc6ea0021bd83a Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 7 Jan 2025 09:45:15 -0700 Subject: [PATCH 20/21] refactor, lint errors --- .../teraslice/src/lib/cluster/services/api.ts | 2 +- .../cluster/backends/native/messaging.ts | 29 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/teraslice/src/lib/cluster/services/api.ts b/packages/teraslice/src/lib/cluster/services/api.ts index 4f950399098..45c83eb5f27 100644 --- a/packages/teraslice/src/lib/cluster/services/api.ts +++ b/packages/teraslice/src/lib/cluster/services/api.ts @@ -137,7 +137,7 @@ export class ApiService { } private async _assetRedirect(req: TerasliceRequest, res: TerasliceResponse) { - const options: OptionsInit & { isStream: true } = { + const options: OptionsInit & { isStream: true } = { prefixUrl: this.assetsUrl, headers: req.headers, searchParams: req.query as Record, diff --git a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts index 3ce1de54762..8a793287bc5 100644 --- a/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts +++ b/packages/teraslice/src/lib/cluster/services/cluster/backends/native/messaging.ts @@ -318,31 +318,33 @@ export class Messaging { }); } - private _createIOServer(server?: HttpServer | HttpsServer, port?: string | number) { + private _createIOServer(options: ListenOptions) { + const { server, port } = options; + const opts = { path: '/native-clustering', pingTimeout: this.configTimeout, pingInterval: this.configTimeout + this.networkLatencyBuffer, perMessageDeflate: false, serveClient: false, - } + }; if (server) { this.io = socketIOServer(server, opts); } else if (port) { this.io = socketIOServer(port, opts); } this._attachRoomsSocketIO(); - + this.io.on('connection', (socket: any) => { this.logger.debug('a connection to cluster_master has been made'); this._registerFns(socket); }); } - + listen(options: ListenOptions = {}) { - const { query, server, port } = options; + const { query } = options; this.messsagingOnline = true; - + if (this.config.clients.networkClient) { // node_master, worker this.io = socketIOClient(this.hostURL, { @@ -350,9 +352,9 @@ export class Messaging { path: '/native-clustering', query }); - + this._registerFns(this.io); - + if (this.self === 'node_master') { this.io.on('networkMessage', (networkMsg: any) => { const { message } = networkMsg; @@ -365,14 +367,11 @@ export class Messaging { } }); } - + this.logger.debug('client network connection is online'); - } else if (server) { - // cluster_master - this._createIOServer(server); - } else if (port) { - // test server - this._createIOServer(undefined, port); + } else { + // cluster_master and test processes + this._createIOServer(options); } // TODO: message queuing will be used until formal process lifecycles are implemented From 804673c954a59e8e286d6387e0c5e3cb89ecd46f Mon Sep 17 00:00:00 2001 From: busma13 Date: Tue, 7 Jan 2025 10:12:55 -0700 Subject: [PATCH 21/21] fix yarn.lock --- yarn.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/yarn.lock b/yarn.lock index 8efad45e8bc..7bc12dfa13f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2206,6 +2206,13 @@ lz4-asm "~0.4.2" node-gzip "~1.1.2" +"@terascope/types@~1.3.1": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@terascope/types/-/types-1.3.2.tgz#444006f6e9fbf45fab0e1b45b1894fb06dc48a35" + integrity sha512-pt0793isrLAK0D6A5qJJDrj0mgE1wa1c12xCjIb1c5zacgtJFxdK6VArTEvieqjyDn+DAUY5h9fAAnUsVPnpSg== + dependencies: + prom-client "~15.1.3" + "@terascope/utils@~1.4.0": version "1.4.2" resolved "https://registry.yarnpkg.com/@terascope/utils/-/utils-1.4.2.tgz#de4134651094c4028db16dc670335a5ed0cb5ac2"