Add resource cleanup on ex error scenario (#3641)

This PR makes the following changes: - Adds resource cleanup on ex error scenario - This adds a functionality to the shutdown function on the `ex controller` that checks to see if shutdown was caused by a specific error. Then proceeds to tell the `cluster master` to cleanup the resources. - #2673 --------- Co-authored-by: busma13 <[email protected]>
terascope · Jun 18, 2024 · 4ea324b · 4ea324b
1 parent 1cadc9e
commit 4ea324b
Show file tree

Hide file tree

Showing 24 changed files with 84 additions and 72 deletions.
diff --git a/e2e/package.json b/e2e/package.json
@@ -42,9 +42,9 @@
         "ms": "^2.1.3"
     },
     "devDependencies": {
-        "@terascope/types": "^0.17.2",
+        "@terascope/types": "^0.17.3",
         "bunyan": "^1.8.15",
-        "elasticsearch-store": "^0.84.2",
+        "elasticsearch-store": "^0.84.3",
         "fs-extra": "^11.2.0",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "2.0.0",
+    "version": "2.0.1",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "0.56.2",
+    "version": "0.56.3",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -29,9 +29,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-types": "^0.50.2",
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/data-types": "^0.50.3",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "@types/validator": "^13.11.10",
         "awesome-phonenumber": "^2.70.0",
         "date-fns": "^2.30.0",
@@ -46,7 +46,7 @@
         "uuid": "^9.0.1",
         "valid-url": "^1.0.9",
         "validator": "^13.12.0",
-        "xlucene-parser": "^0.58.2"
+        "xlucene-parser": "^0.58.3"
     },
     "devDependencies": {
         "@types/ip6addr": "^0.2.6",

diff --git a/packages/data-types/package.json b/packages/data-types/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-types",
     "displayName": "Data Types",
-    "version": "0.50.2",
+    "version": "0.50.3",
     "description": "A library for defining the data structures and mapping",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
     "bugs": {
@@ -26,8 +26,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "graphql": "^14.7.0",
         "lodash": "^4.17.21",
         "yargs": "^17.7.2"

diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/elasticsearch-api",
     "displayName": "Elasticsearch API",
-    "version": "3.20.2",
+    "version": "3.20.3",
     "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme",
     "bugs": {
@@ -23,16 +23,16 @@
         "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "bluebird": "^3.7.2",
         "setimmediate": "^1.0.5"
     },
     "devDependencies": {
         "@opensearch-project/opensearch": "^1.2.0",
         "@types/elasticsearch": "^5.0.43",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.84.2",
+        "elasticsearch-store": "^0.84.3",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
         "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0"

diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json
@@ -1,7 +1,7 @@
 {
     "name": "elasticsearch-store",
     "displayName": "Elasticsearch Store",
-    "version": "0.84.2",
+    "version": "0.84.3",
     "description": "An API for managing an elasticsearch index, with versioning and migration support.",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme",
     "bugs": {
@@ -29,10 +29,10 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^0.56.2",
-        "@terascope/data-types": "^0.50.2",
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/data-mate": "^0.56.3",
+        "@terascope/data-types": "^0.50.3",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "ajv": "^6.12.6",
         "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0",
         "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0",
@@ -41,7 +41,7 @@
         "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1",
         "setimmediate": "^1.0.5",
         "uuid": "^9.0.1",
-        "xlucene-translator": "^0.44.2"
+        "xlucene-translator": "^0.44.3"
     },
     "devDependencies": {
         "@types/uuid": "^9.0.8"

diff --git a/packages/job-components/package.json b/packages/job-components/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/job-components",
     "displayName": "Job Components",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
     "bugs": {
@@ -32,8 +32,8 @@
         "test:watch": "NODE_OPTIONS='--experimental-vm-modules' ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "convict": "^6.2.4",
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",

diff --git a/packages/scripts/package.json b/packages/scripts/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "0.77.2",
+    "version": "0.77.3",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {
@@ -32,7 +32,7 @@
     },
     "dependencies": {
         "@kubernetes/client-node": "^0.20.0",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/utils": "^0.59.3",
         "codecov": "^3.8.3",
         "execa": "^5.1.0",
         "fs-extra": "^11.2.0",

diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json
@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -29,15 +29,15 @@
     },
     "dependencies": {
         "@terascope/file-asset-apis": "^0.13.0",
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "bluebird": "^3.7.2",
         "bunyan": "^1.8.15",
         "convict": "^6.2.4",
         "convict-format-with-moment": "^6.2.0",
         "convict-format-with-validator": "^6.2.0",
         "elasticsearch": "^15.4.1",
-        "elasticsearch-store": "^0.84.2",
+        "elasticsearch-store": "^0.84.3",
         "express": "^4.19.2",
         "js-yaml": "^4.1.0",
         "nanoid": "^3.3.4",

diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-cli",
     "displayName": "Teraslice CLI",
-    "version": "2.0.0",
+    "version": "2.0.1",
     "description": "Command line manager for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "teraslice"
@@ -38,8 +38,8 @@
     },
     "dependencies": {
         "@terascope/fetch-github-release": "^0.8.10",
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "chalk": "^4.1.2",
         "cli-table3": "^0.6.4",
         "easy-table": "^1.2.0",
@@ -54,7 +54,7 @@
         "pretty-bytes": "^5.6.0",
         "prompts": "^2.4.2",
         "signale": "^1.4.0",
-        "teraslice-client-js": "^0.60.2",
+        "teraslice-client-js": "^0.60.3",
         "tmp": "^0.2.0",
         "tty-table": "^4.2.3",
         "yargs": "^17.7.2",

diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-client-js",
     "displayName": "Teraslice Client (JavaScript)",
-    "version": "0.60.2",
+    "version": "0.60.3",
     "description": "A Node.js client for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "elasticsearch",
@@ -32,8 +32,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "auto-bind": "^4.0.0",
         "got": "^11.8.3"
     },

diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/teraslice-messaging",
     "displayName": "Teraslice Messaging",
-    "version": "1.0.0",
+    "version": "1.0.1",
     "description": "An internal teraslice messaging library using socket.io",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme",
     "bugs": {
@@ -35,8 +35,8 @@
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "ms": "^2.1.3",
         "nanoid": "^3.3.4",
         "p-event": "^4.2.0",

diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/teraslice-state-storage",
     "displayName": "Teraslice State Storage",
-    "version": "0.53.2",
+    "version": "0.53.3",
     "description": "State storage operation api for teraslice",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme",
     "bugs": {
@@ -23,8 +23,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^3.20.2",
-        "@terascope/utils": "^0.59.2"
+        "@terascope/elasticsearch-api": "^3.20.3",
+        "@terascope/utils": "^0.59.3"
     },
     "engines": {
         "node": ">=14.17.0",

diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json
@@ -37,10 +37,10 @@
         "fs-extra": "^11.2.0"
     },
     "devDependencies": {
-        "@terascope/job-components": "^1.0.0"
+        "@terascope/job-components": "^1.0.1"
     },
     "peerDependencies": {
-        "@terascope/job-components": ">=1.0.0"
+        "@terascope/job-components": ">=1.0.1"
     },
     "engines": {
         "node": ">=14.17.0",

diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "2.0.0",
+    "version": "2.0.1",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {
@@ -38,11 +38,11 @@
         "ms": "^2.1.3"
     },
     "dependencies": {
-        "@terascope/elasticsearch-api": "^3.20.2",
-        "@terascope/job-components": "^1.0.0",
-        "@terascope/teraslice-messaging": "^1.0.0",
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/elasticsearch-api": "^3.20.3",
+        "@terascope/job-components": "^1.0.1",
+        "@terascope/teraslice-messaging": "^1.0.1",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "async-mutex": "^0.5.0",
         "barbe": "^3.0.16",
         "body-parser": "^1.20.2",
@@ -63,7 +63,7 @@
         "semver": "^7.6.2",
         "socket.io": "^1.7.4",
         "socket.io-client": "^1.7.4",
-        "terafoundation": "^1.0.0",
+        "terafoundation": "^1.0.1",
         "uuid": "^9.0.1"
     },
     "devDependencies": {

diff --git a/packages/teraslice/src/lib/workers/execution-controller/index.ts b/packages/teraslice/src/lib/workers/execution-controller/index.ts
@@ -404,7 +404,19 @@ export class ExecutionController {
         await this._endExecution();
     }
 
-    async shutdown(block = true) {
+    async shutdown(eventType?: string, shutdownError?: Error, block: boolean = true) {
+        if (eventType === 'error' && shutdownError) {
+            /// Add errors to this list as needed. Errors not in this list won't cleanup resources
+            const errorList = [
+                'index specified in reader does not exist'
+            ];
+            /// Tell cluster_master that shutdown is due to a specific error
+            /// Cleans up kubernetes resources. For native, kills processes
+            if (errorList.includes(shutdownError.message)) {
+                this.logger.warn('sent request to cluster_master to cleanup job resources.');
+                await this.client.sendExecutionFinished(shutdownError.message);
+            }
+        }
         if (this.isShutdown) return;
         if (!this.isInitialized) return;
         if (this.isShuttingDown) {

diff --git a/packages/teraslice/src/lib/workers/worker/index.ts b/packages/teraslice/src/lib/workers/worker/index.ts
@@ -234,7 +234,7 @@ export class Worker {
         this.slicesProcessed += 1;
     }
 
-    async shutdown(block?: boolean, event?: string, shutdownError?: Error) {
+    async shutdown(event?: string, shutdownError?: Error, block?: boolean) {
         if (this.isShutdown) return;
         if (!this.isInitialized) return;
         const { exId } = this.executionContext;

diff --git a/packages/teraslice/worker-service.js b/packages/teraslice/worker-service.js
@@ -73,7 +73,7 @@ async function main() {
 
     cmd.shutdownHandler = shutdownHandler(context, (event, err) => {
         if (!cmd.instance) return Promise.resolve();
-        return cmd.instance.shutdown(true, event, err);
+        return cmd.instance.shutdown(event, err, true);
     });
 
     try {

diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json
@@ -1,7 +1,7 @@
 {
     "name": "ts-transforms",
     "displayName": "TS Transforms",
-    "version": "0.85.2",
+    "version": "0.85.3",
     "description": "An ETL framework built upon xlucene-evaluator",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme",
     "bugs": {
@@ -35,9 +35,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-mate": "^0.56.2",
-        "@terascope/types": "^0.17.2",
-        "@terascope/utils": "^0.59.2",
+        "@terascope/data-mate": "^0.56.3",
+        "@terascope/types": "^0.17.3",
+        "@terascope/utils": "^0.59.3",
         "awesome-phonenumber": "^2.70.0",
         "graphlib": "^2.1.8",
         "is-ip": "^3.1.0",