diff --git a/e2e/package.json b/e2e/package.json index 3e50b7fea2c..d5d09bf2ed5 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -45,7 +45,7 @@ "devDependencies": { "@terascope/types": "^1.2.0", "bunyan": "^1.8.15", - "elasticsearch-store": "^1.3.2", + "elasticsearch-store": "^1.3.3", "fs-extra": "^11.2.0", "ms": "^2.1.3", "nanoid": "^5.0.7", diff --git a/e2e/test/cases/cluster/storage-spec.ts b/e2e/test/cases/cluster/storage-spec.ts new file mode 100644 index 00000000000..a9f93f131cd --- /dev/null +++ b/e2e/test/cases/cluster/storage-spec.ts @@ -0,0 +1,29 @@ +import { TerasliceHarness } from '../../teraslice-harness.js'; +import { TEST_OPENSEARCH } from '../../config.js'; + +describe('mappings', () => { + let terasliceHarness: TerasliceHarness; + + beforeAll(async () => { + terasliceHarness = new TerasliceHarness(); + await terasliceHarness.init(); + await terasliceHarness.resetState(); + }); + + it('should have a jobs index with dynamic mapping false', async () => { + const mapping = await terasliceHarness.client.indices.getMapping({ index: '*__jobs' }); + const indexName = Object.keys(mapping)[0]; + const searchVersion = terasliceHarness.client.__meta.version; + if (!TEST_OPENSEARCH && searchVersion.charAt(0) === '6') { + expect(mapping[indexName]).toMatchObject({ + mappings: { + job: expect.objectContaining({ dynamic: 'false' }) + } + }); + } else { + expect(mapping[indexName]).toMatchObject({ + mappings: expect.objectContaining({ dynamic: 'false' }) + }); + } + }); +}); diff --git a/e2e/test/config.ts b/e2e/test/config.ts index 6d1b6ed29b9..cf94021a70f 100644 --- a/e2e/test/config.ts +++ b/e2e/test/config.ts @@ -111,5 +111,6 @@ export { MINIO_ACCESS_KEY, MINIO_SECRET_KEY, ENCRYPT_MINIO, - ROOT_CERT_PATH + ROOT_CERT_PATH, + TEST_OPENSEARCH }; diff --git a/package.json b/package.json index ebb4af05d8a..a506dc99aac 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "2.6.2", + "version": "2.6.3", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index 493fb3f65d2..4756f200132 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -33,7 +33,7 @@ "@opensearch-project/opensearch": "^1.2.0", "@types/elasticsearch": "^5.0.43", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.3.2", + "elasticsearch-store": "^1.3.3", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index bf6cdad9166..e7cd1c3515a 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "1.3.2", + "version": "1.3.3", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { diff --git a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts index dc3be5c9014..ad45b8cf665 100644 --- a/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts +++ b/packages/elasticsearch-store/src/elasticsearch-client/method-helpers/helper-utils.ts @@ -34,9 +34,11 @@ export function ensureNoTypeInMapping(mappings: Record | undefined) for (const [k, v] of Object.entries(mappings)) { if (k === 'properties') parsed[k] = v; if (k === '_meta') parsed[k] = v; + if (k === 'dynamic') parsed[k] = v; if (v.properties) parsed.properties = v.properties; if (v._meta) parsed._meta = v._meta; + if (v.dynamic !== undefined) parsed.dynamic = v.dynamic; } } return parsed; diff --git a/packages/elasticsearch-store/test/method-helpers-spec.ts b/packages/elasticsearch-store/test/method-helpers-spec.ts new file mode 100644 index 00000000000..0ad3dd2a4a1 --- /dev/null +++ b/packages/elasticsearch-store/test/method-helpers-spec.ts @@ -0,0 +1,133 @@ +import { ensureNoTypeInMapping } from '../src/elasticsearch-client/method-helpers/index.js'; + +describe('test method helpers', () => { + describe('->ensureNoTypeInMapping', () => { + describe('mappings with type', () => { + const mappingsWithType = { + myType: { + _meta: { + foo: 'foo' + }, + _all: { + enabled: false + }, + dynamic: false, + properties: { + _context: { + type: 'keyword' + }, + _created: { + type: 'date' + }, + _updated: { + type: 'date' + }, + _deleted: { + type: 'boolean' + }, + _deleted_on: { + type: 'date' + } + } + } + }; + + const newMappings = ensureNoTypeInMapping(mappingsWithType); + + it('should remove myType wrapper object from mappingsWithType', () => { + expect(newMappings.myType).toBe(undefined); + }); + + it('should remove _all from mappingsWithType', () => { + expect(newMappings._all).toBe(undefined); + }); + + it('should include properties, _meta, and dynamic from mappingsWithType', () => { + expect(newMappings).toMatchObject({ + _meta: { + foo: 'foo' + }, + dynamic: false, + properties: { + _context: { + type: 'keyword', + }, + _created: { + type: 'date', + }, + _deleted: { + type: 'boolean', + }, + _deleted_on: { + type: 'date', + }, + _updated: { + type: 'date', + }, + }, + }); + }); + }); + + describe('mappings without type', () => { + const mappingsWithoutType = { + _meta: { + foo: 'foo' + }, + _all: { + enabled: false + }, + dynamic: false, + properties: { + _context: { + type: 'keyword' + }, + _created: { + type: 'date' + }, + _updated: { + type: 'date' + }, + _deleted: { + type: 'boolean' + }, + _deleted_on: { + type: 'date' + } + } + }; + + const newMappings = ensureNoTypeInMapping(mappingsWithoutType); + + it('should remove _all from mappingsWithoutType', () => { + expect(newMappings._all).toBe(undefined); + }); + + it('should include properties, _meta, and dynamic from mappingsWithoutType', () => { + expect(newMappings).toMatchObject({ + _meta: { + foo: 'foo' + }, + dynamic: false, + properties: { + _context: { + type: 'keyword', + }, + _created: { + type: 'date', + }, + _deleted: { + type: 'boolean', + }, + _deleted_on: { + type: 'date', + }, + _updated: { + type: 'date', + }, + }, + }); + }); + }); + }); +}); diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index e1ddb161229..c2f2f8f20a9 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "1.5.3", + "version": "1.5.4", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -37,7 +37,7 @@ "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^1.3.2", + "elasticsearch-store": "^1.3.3", "express": "^4.21.0", "js-yaml": "^4.1.0", "nanoid": "^5.0.7", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 665aabb88ff..972fcfd3090 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "2.6.2", + "version": "2.6.3", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { @@ -63,7 +63,7 @@ "semver": "^7.6.3", "socket.io": "^1.7.4", "socket.io-client": "^1.7.4", - "terafoundation": "^1.5.3", + "terafoundation": "^1.5.4", "uuid": "^10.0.0" }, "devDependencies": {