Rearchitecture backend (#193)

* BREAKING CHANGE: trigger v2 release

.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -10,12 +10,8 @@ src/python/.idea
 npm-debug.log.*
 .idea
 .coverage
-index.js
-index.js.map
-pyShellType.js
-pyShellType.js.map
-*.test.js
-*.test.js.map
+*.js
+*.js.map
 *.d.ts
 .pytest_cache
 .vscode/settings.json

.vscode/launch.json

@@ -53,8 +53,7 @@
             "request": "launch",
             "program": "${workspaceRoot}/node_modules/mocha/bin/_mocha",
             "args": ["-r", "ts-node/register", "${relativeFile}","--ui","tdd","--no-timeouts"],
-            "cwd": "${workspaceRoot}",
-            "protocol": "inspector"
+            "cwd": "${workspaceRoot}"
           }
     ]
 }

index.ts → PythonExecutor.ts

@@ -1,6 +1,6 @@
 import { PythonShell, Options, NewlineTransformer } from 'python-shell'
 import { EOL } from 'os'
-import { Readable } from 'stream'
+import { randomBytes } from 'crypto'
 
 export interface FrameSummary {
 	_line: string
@@ -51,23 +51,32 @@ export interface PythonResult {
 	internalError: string,
 	caller: string,
 	lineno: number,
-	done: boolean
+	done: boolean,
+	startResult: boolean,
+	evaluatorName: string,
 }
 
-export class PythonEvaluator {
-	private static readonly areplPythonBackendFolderPath = __dirname + '/python/'
-
-	/**
-	 * whether python is busy executing inputted code
-	 */
-	executing = false
+/**
+ * Starting = Starting or restarting. 
+ * Ending = Process is exiting. 
+ * Executing = Executing inputted code. 
+ * DirtyFree = evaluator may have been polluted by side-effects from previous code, but is free for more code. 
+ * FreshFree = evaluator is ready for the first run of code
+ */
+export enum PythonState {
+	Starting,
+	Ending,
+	Executing,
+	DirtyFree,
+	FreshFree
+}
 
-	/**
-	 * whether python backend process is running / not running
-	 */
-	running = false
+export class PythonExecutor {
+	private static readonly areplPythonBackendFolderPath = __dirname + '/python/'
 
-	restarting = false
+	state: PythonState = PythonState.Starting
+	finishedStartingCallback: Function
+	evaluatorName: string
 	private startTime: number
 
 	/**
@@ -98,16 +107,21 @@ export class PythonEvaluator {
 		this.options.mode = 'binary'
 		this.options.stdio = ['pipe', 'pipe', 'pipe', 'pipe']
 		if (!options.pythonPath) this.options.pythonPath = PythonShell.defaultPythonPath
-		if (!options.scriptPath) this.options.scriptPath = PythonEvaluator.areplPythonBackendFolderPath
+		if (!options.scriptPath) this.options.scriptPath = PythonExecutor.areplPythonBackendFolderPath
+
+		this.evaluatorName = randomBytes(16).toString('hex')
 	}
 
 
 	/**
 	 * does not do anything if program is currently executing code 
 	 */
 	execCode(code: ExecArgs) {
-		if (this.executing) return
-		this.executing = true
+		if (this.state == PythonState.Executing){
+			console.error('Incoming code detected while process is still executing. \
+			This should never happen')
+		}
+		this.state = PythonState.Executing
 		this.startTime = Date.now()
 		this.pyshell.send(JSON.stringify(code) + EOL)
 	}
@@ -125,64 +139,58 @@ export class PythonEvaluator {
 	 */
 	restart(callback = () => { }) {
 
-		this.restarting = false
+		this.state = PythonState.Ending
 
 		// register callback for restart
 		// using childProcess callback instead of pyshell callback
 		// (pyshell callback only happens when process exits voluntarily)
 		this.pyshell.childProcess.on('exit', () => {
-			this.restarting = true
-			this.executing = false
-			this.start()
-			callback()
+			this.start(callback)
 		})
 
 		this.stop()
 	}
 
 	/**
-	 * kills python process.  force-kills if necessary after 50ms.
-	 * you can check python_evaluator.running to see if process is dead yet
+	 * Kills python process.  Force-kills if necessary after 50ms.
+	 * You can check python_evaluator.running to see if process is dead yet
 	 */
-	stop() {
-		// pyshell has 50 ms to die gracefully
-		this.pyshell.childProcess.kill()
-		this.running = !this.pyshell.childProcess.killed
-		if (this.running) console.info("pyshell refused to die")
-		else this.executing = false
-
-		setTimeout(() => {
-			if (this.running && !this.restarting) {
-				// murder the process with extreme prejudice
-				this.pyshell.childProcess.kill('SIGKILL')
-				if (this.pyshell.childProcess.killed) {
-					console.error("the python process simply cannot be killed!")
+	stop(kill_immediately=false) {
+		this.state = PythonState.Ending
+		const kill_signal = kill_immediately ? 'SIGKILL' : 'SIGTERM'
+		this.pyshell.childProcess.kill(kill_signal)
+
+		if(!kill_immediately){
+			// pyshell has 50 ms to die gracefully
+			setTimeout(() => {
+				if (this.state == PythonState.Ending) {
+					// python didn't respect the SIGTERM, force-kill it
+					this.pyshell.childProcess.kill('SIGKILL')
 				}
-				else this.executing = false
-			}
-		}, 50)
+			}, 50)
+		}
 	}
 
 	/**
 	 * starts python_evaluator.py. Will NOT WORK with python 2
 	 */
-	start() {
+	start(finishedStartingCallback) {
+		this.state = PythonState.Starting
 		console.log("Starting Python...")
+		this.finishedStartingCallback = finishedStartingCallback
+		this.startTime = Date.now()
 		this.pyshell = new PythonShell('arepl_python_evaluator.py', this.options)
 
 		const resultPipe = this.pyshell.childProcess.stdio[3]
 		const newlineTransformer = new NewlineTransformer()
 		resultPipe.pipe(newlineTransformer).on('data', this.handleResult.bind(this))
 
-		// not sure why exactly I have to wrap onPrint/onStderr w/ lambda
-		// but tests fail if I don't
 		this.pyshell.stdout.on('data', (message: Buffer) => {
 			this.onPrint(message.toString())
 		})
 		this.pyshell.stderr.on('data', (log: Buffer) => {
 			this.onStderr(log.toString())
 		})
-		this.running = true
 	}
 
 	/**
@@ -220,12 +228,22 @@ export class PythonEvaluator {
 			internalError: "",
 			caller: "",
 			lineno: -1,
-			done: true
+			done: true,
+			startResult: false,
+			evaluatorName: this.evaluatorName
 		}
 
 		try {
 			pyResult = JSON.parse(results)
-			this.executing = !pyResult['done']
+			if(pyResult.startResult){
+				console.log(`Finished starting in ${Date.now() - this.startTime}`)
+				this.state = PythonState.FreshFree
+				this.finishedStartingCallback()
+				return
+			}
+			if(pyResult['done'] == true){
+				this.state = PythonState.DirtyFree
+			}
 
 			pyResult.execTime = pyResult.execTime * 1000 // convert into ms
 			pyResult.totalPyTime = pyResult.totalPyTime * 1000
@@ -258,40 +276,14 @@ export class PythonEvaluator {
 		return PythonShell.checkSyntax(code);
 	}
 
-	/**
-	 * checks syntax without executing code
-	 * @param {string} filePath
-	 * @returns {Promise} rejects w/ stderr if syntax failure
-	 */
-	async checkSyntaxFile(filePath: string) {
-		// note that this should really be done in python_evaluator.py
-		// but communication with that happens through just one channel (stdin/stdout)
-		// so for now i prefer to keep this seperate
-
-		return PythonShell.checkSyntaxFile(filePath);
-	}
-
 	/**
 	 * gets rid of unnecessary File "<string>" message in exception
 	 * @example err:
 	 * Traceback (most recent call last):\n  File "<string>", line 1, in <module>\nNameError: name \'x\' is not defined\n
 	 */
-	formatPythonException(err: string) {
+	private formatPythonException(err: string) {
 		//replace File "<string>" (pointless)
 		err = err.replace(/File \"<string>\", /g, "")
 		return err
 	}
-
-	/**
-	 * delays execution of function by ms milliseconds, resetting clock every time it is called
-	 * Useful for real-time execution so execCode doesn't get called too often
-	 * thanks to https://stackoverflow.com/a/1909508/6629672
-	 */
-	debounce = (function () {
-		let timer: any = 0;
-		return function (callback, ms: number, ...args: any[]) {
-			clearTimeout(timer);
-			timer = setTimeout(callback, ms, args);
-		};
-	})();
 }

README.md

@@ -12,7 +12,7 @@ Although it is meant for AREPL, it is not dependent upon AREPL and can be used b
 
 > npm install [arepl-backend](https://www.npmjs.com/package/arepl-backend)
 
-must have python 3.7 or greater
+Must have python 3.7 or greater
 
 ## Usage
 
@@ -35,35 +35,41 @@ Semantic release cheatsheet:
 
 #### Table of Contents
 
+*   [PythonState](#pythonstate)
 *   [constructor](#constructor)
     *   [Parameters](#parameters)
-*   [executing](#executing)
-*   [running](#running)
-*   [debounce](#debounce)
 *   [execCode](#execcode)
     *   [Parameters](#parameters-1)
 *   [sendStdin](#sendstdin)
     *   [Parameters](#parameters-2)
 *   [restart](#restart)
     *   [Parameters](#parameters-3)
 *   [stop](#stop)
-*   [start](#start)
-*   [onResult](#onresult)
     *   [Parameters](#parameters-4)
-*   [onPrint](#onprint)
+*   [start](#start)
     *   [Parameters](#parameters-5)
-*   [onStderr](#onstderr)
+*   [onResult](#onresult)
     *   [Parameters](#parameters-6)
-*   [handleResult](#handleresult)
+*   [onPrint](#onprint)
     *   [Parameters](#parameters-7)
-*   [checkSyntax](#checksyntax)
+*   [onStderr](#onstderr)
     *   [Parameters](#parameters-8)
-*   [checkSyntaxFile](#checksyntaxfile)
+*   [handleResult](#handleresult)
     *   [Parameters](#parameters-9)
-*   [formatPythonException](#formatpythonexception)
+*   [checkSyntax](#checksyntax)
     *   [Parameters](#parameters-10)
+*   [formatPythonException](#formatpythonexception)
+    *   [Parameters](#parameters-11)
     *   [Examples](#examples)
 
+### PythonState
+
+Starting = Starting or restarting.
+Ending = Process is exiting.
+Executing = Executing inputted code.
+DirtyFree = evaluator may have been polluted by side-effects from previous code, but is free for more code.
+FreshFree = evaluator is ready for the first run of code
+
 ### constructor
 
 starts python\_evaluator.py
@@ -72,20 +78,6 @@ starts python\_evaluator.py
 
 *   `options`  Process / Python options. If not specified sensible defaults are inferred. (optional, default `{}`)
 
-### executing
-
-whether python is busy executing inputted code
-
-### running
-
-whether python backend process is running / not running
-
-### debounce
-
-delays execution of function by ms milliseconds, resetting clock every time it is called
-Useful for real-time execution so execCode doesn't get called too often
-thanks to <https://stackoverflow.com/a/1909508/6629672>
-
 ### execCode
 
 does not do anything if program is currently executing code
@@ -111,13 +103,21 @@ After process restarts the callback passed in is invoked
 
 ### stop
 
-kills python process.  force-kills if necessary after 50ms.
-you can check python\_evaluator.running to see if process is dead yet
+Kills python process.  Force-kills if necessary after 50ms.
+You can check python\_evaluator.running to see if process is dead yet
+
+#### Parameters
+
+*   `kill_immediately`   (optional, default `false`)
 
 ### start
 
 starts python\_evaluator.py. Will NOT WORK with python 2
 
+#### Parameters
+
+*   `finishedStartingCallback` &#x20;
+
 ### onResult
 
 Overwrite this with your own handler.
@@ -163,16 +163,6 @@ checks syntax without executing code
 
 Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** rejects w/ stderr if syntax failure
 
-### checkSyntaxFile
-
-checks syntax without executing code
-
-#### Parameters
-
-*   `filePath` **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)**&#x20;
-
-Returns **[Promise](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise)** rejects w/ stderr if syntax failure
-
 ### formatPythonException
 
 gets rid of unnecessary File "<string>" message in exception