From 20e9014b6d8d551699de0a1f9bf94ff6e38a41d1 Mon Sep 17 00:00:00 2001 From: Erfan Besharat Date: Sun, 8 Dec 2024 17:04:40 +0100 Subject: [PATCH] Initial setup for concurrent workers --- package-lock.json | 2 +- packages/cli/src/api/split.ts | 31 +-- packages/cli/src/commands/split.ts | 31 +-- packages/cli/src/languagesPlugins/types.ts | 6 +- packages/cli/src/splitRunner/splitRunner.ts | 228 ++------------------ packages/cli/src/splitRunner/worker.ts | 188 ++++++++++++++++ 6 files changed, 250 insertions(+), 236 deletions(-) create mode 100644 packages/cli/src/splitRunner/worker.ts diff --git a/package-lock.json b/package-lock.json index 1518b99..520f15e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12334,7 +12334,7 @@ }, "packages/cli": { "name": "@nanoapi.io/napi", - "version": "0.0.19", + "version": "0.0.20", "dependencies": { "axios": "^1.7.7", "express": "^4.21.1", diff --git a/packages/cli/src/api/split.ts b/packages/cli/src/api/split.ts index dc90c4e..04e52da 100644 --- a/packages/cli/src/api/split.ts +++ b/packages/cli/src/api/split.ts @@ -1,11 +1,11 @@ import fs from "fs"; import path from "path"; +import { z } from "zod"; import DependencyTreeManager from "../dependencyManager/dependencyManager"; +import { Group } from "../dependencyManager/types"; import { cleanupOutputDir, createOutputDir } from "../helper/file"; import SplitRunner from "../splitRunner/splitRunner"; import { splitSchema } from "./helpers/validation"; -import { z } from "zod"; -import { Group } from "../dependencyManager/types"; export function split(payload: z.infer) { console.time("split command"); @@ -13,7 +13,7 @@ export function split(payload: z.infer) { // Get the dependency tree const dependencyTreeManager = new DependencyTreeManager( - payload.entrypointPath, + payload.entrypointPath ); const outputDir = payload.outputDir || path.dirname(payload.entrypointPath); @@ -33,15 +33,22 @@ export function split(payload: z.infer) { const targetDir = path.dirname(payload.entrypointPath); const annotationDirectory = path.join(outputDir, index.toString()); - files.forEach((file) => { - const relativeFileNamePath = path.relative(targetDir, file.path); - const destinationPath = path.join( - annotationDirectory, - relativeFileNamePath, - ); - fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); - fs.writeFileSync(destinationPath, file.sourceCode, "utf8"); - }); + files + .then((files) => { + files.forEach((file) => { + const relativeFileNamePath = path.relative(targetDir, file.path); + const destinationPath = path.join( + annotationDirectory, + relativeFileNamePath + ); + fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); + fs.writeFileSync(destinationPath, file.sourceCode, "utf8"); + }); + }) + .catch((error) => { + console.error(error); + throw error; + }); }); // Store the processed annotations in the output directory diff --git a/packages/cli/src/commands/split.ts b/packages/cli/src/commands/split.ts index fd45e1a..bf99b1b 100644 --- a/packages/cli/src/commands/split.ts +++ b/packages/cli/src/commands/split.ts @@ -1,13 +1,13 @@ -import path from "path"; import fs from "fs"; +import path from "path"; import DependencyTreeManager from "../dependencyManager/dependencyManager"; +import { Group } from "../dependencyManager/types"; import { cleanupOutputDir, createOutputDir } from "../helper/file"; import SplitRunner from "../splitRunner/splitRunner"; -import { Group } from "../dependencyManager/types"; export default function splitCommandHandler( entrypointPath: string, // Path to the entrypoint file - outputDir: string, // Path to the output directory + outputDir: string // Path to the output directory ) { const groupMap: Record = {}; @@ -27,15 +27,22 @@ export default function splitCommandHandler( const targetDir = path.dirname(entrypointPath); const annotationDirectory = path.join(outputDir, index.toString()); - files.forEach((file) => { - const relativeFileNamePath = path.relative(targetDir, file.path); - const destinationPath = path.join( - annotationDirectory, - relativeFileNamePath, - ); - fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); - fs.writeFileSync(destinationPath, file.sourceCode, "utf8"); - }); + files + .then((files) => { + files.forEach((file) => { + const relativeFileNamePath = path.relative(targetDir, file.path); + const destinationPath = path.join( + annotationDirectory, + relativeFileNamePath + ); + fs.mkdirSync(path.dirname(destinationPath), { recursive: true }); + fs.writeFileSync(destinationPath, file.sourceCode, "utf8"); + }); + }) + .catch((error) => { + console.error(error); + throw error; + }); }); // Store the processed annotations in the output directory diff --git a/packages/cli/src/languagesPlugins/types.ts b/packages/cli/src/languagesPlugins/types.ts index 5b6ac2b..bb0a4ea 100644 --- a/packages/cli/src/languagesPlugins/types.ts +++ b/packages/cli/src/languagesPlugins/types.ts @@ -1,5 +1,5 @@ -import { Group } from "../dependencyManager/types"; import Parser from "tree-sitter"; +import { Group } from "../dependencyManager/types"; export interface DepImportIdentifier { // Specific to each programing languages. Used by the language plugins. @@ -61,7 +61,7 @@ export interface LanguagePlugin { removeAnnotationFromOtherGroups( sourceCode: string, - groupToKeep: Group, + groupToKeep: Group ): string; getImports(filePath: string, node: Parser.SyntaxNode): DepImport[]; @@ -71,7 +71,7 @@ export interface LanguagePlugin { cleanupInvalidImports( filePath: string, sourceCode: string, - exportMap: Map, + exportMap: Map ): string; cleanupUnusedImports(filePath: string, sourceCode: string): string; diff --git a/packages/cli/src/splitRunner/splitRunner.ts b/packages/cli/src/splitRunner/splitRunner.ts index 75b3f55..d97bc0d 100644 --- a/packages/cli/src/splitRunner/splitRunner.ts +++ b/packages/cli/src/splitRunner/splitRunner.ts @@ -1,231 +1,43 @@ -import { Group } from "../dependencyManager/types"; -import { removeIndexesFromSourceCode } from "../helper/file"; +import path from "path"; +import { Worker } from "worker_threads"; import DependencyTreeManager from "../dependencyManager/dependencyManager"; +import { Group } from "../dependencyManager/types"; import { File } from "./types"; -import Parser from "tree-sitter"; -import assert from "assert"; -import { getLanguagePlugin } from "../languagesPlugins"; -import { DepExport } from "../languagesPlugins/types"; class SplitRunner { private dependencyTreeManager: DependencyTreeManager; - private entrypointPath: string; private group: Group; - private files: File[]; constructor(dependencyTreeManager: DependencyTreeManager, group: Group) { this.dependencyTreeManager = dependencyTreeManager; - this.entrypointPath = dependencyTreeManager.dependencyTree.path; this.group = group; - this.files = dependencyTreeManager.getFiles(); - } - - #removeAnnotationFromOtherGroups() { - this.files = this.files.map((file) => { - const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path); - - const updatedSourceCode = languagePlugin.removeAnnotationFromOtherGroups( - file.sourceCode, - this.group, - ); - return { ...file, sourceCode: updatedSourceCode }; - }); - } - - #getExportMap() { - const exportMap = new Map(); - - this.files.forEach((file) => { - const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path); - - const tree = languagePlugin.parser.parse(file.sourceCode); - - const exports = languagePlugin.getExports(tree.rootNode); - - exportMap.set(file.path, exports); - }); - - return exportMap; - } - - #removeInvalidImportsAndUsages(exportMap: Map) { - this.files = this.files.map((file) => { - const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path); - - const updatedSourceCode = languagePlugin.cleanupInvalidImports( - file.path, - file.sourceCode, - exportMap, - ); - - return { ...file, sourceCode: updatedSourceCode }; - }); } - #removeUnusedImports() { - this.files = this.files.map((file) => { - const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path); + async run(): Promise { + console.time("\nSplitting"); - const updatedSourceCode = languagePlugin.cleanupUnusedImports( - file.path, - file.sourceCode, - ); - - return { ...file, sourceCode: updatedSourceCode }; + const worker = new Worker(path.resolve(__dirname, "worker.js"), { + workerData: { + entrypointPath: this.dependencyTreeManager.dependencyTree.path, + group: this.group, + files: this.dependencyTreeManager.getFiles(), + }, }); - } - - #removeUnusedFiles() { - let fileRemoved = true; - while (fileRemoved) { - fileRemoved = false; - - // We always want to keep the entrypoint file. - // It will never be imported anywhere, so we add it now. - const filesToKeep = new Set(); - filesToKeep.add(this.dependencyTreeManager.dependencyTree.path); - - this.files.forEach((file) => { - const languagePlugin = getLanguagePlugin( - this.entrypointPath, - file.path, - ); - - const tree = languagePlugin.parser.parse(file.sourceCode); - - const imports = languagePlugin.getImports(file.path, tree.rootNode); - - imports.forEach((depImport) => { - if (depImport.isExternal || !depImport.source) { - // Ignore external dependencies - return; - } - - filesToKeep.add(depImport.source); - }); - }); - - const previousFilesLength = this.files.length; - this.files = this.files.filter((file) => { - return filesToKeep.has(file.path); + return new Promise((resolve, reject) => { + worker.on("message", (updatedFiles: File[]) => { + console.timeEnd("Splitting"); + resolve(updatedFiles); }); - if (this.files.length !== previousFilesLength) { - fileRemoved = true; - } - } - } - - #removeUnusedExports(exportMap: Map) { - let exportDeleted = true; - while (exportDeleted) { - exportDeleted = false; - - // const usedExportMap = new Map(); - - this.files = this.files.map((file) => { - const languagePlugin = getLanguagePlugin( - this.entrypointPath, - file.path, - ); - - const tree = languagePlugin.parser.parse(file.sourceCode); - - const imports = languagePlugin.getImports(file.path, tree.rootNode); - - imports.forEach((depImport) => { - if (depImport.isExternal || !depImport.source) { - // Ignore external dependencies - return; - } - - // for each import, reconstruct the export map - const depExport = exportMap.get(depImport.source); - if (!depExport) { - throw new Error("Export not found"); - } - - // check named imports - }); - - return file; - }); - } - // TODO - // Step 1, create variable to track which export is used - // Step 2, iterate over all file imports. If the import is used, mark the export as used - // Step 3, iterate over each file, and remove the unused exports - - // Repeat above step until no more unused exports are found - assert(exportMap); - } - - #removeErrors() { - this.files = this.files.map((file) => { - const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path); - - const tree = languagePlugin.parser.parse(file.sourceCode); - - const indexesToRemove: { startIndex: number; endIndex: number }[] = []; - - const query = new Parser.Query( - languagePlugin.parser.getLanguage(), - "(ERROR) @error", - ); - const errorCaptures = query.captures(tree.rootNode); - errorCaptures.forEach((capture) => { - indexesToRemove.push({ - startIndex: capture.node.startIndex, - endIndex: capture.node.endIndex, - }); + worker.on("error", reject); + worker.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`Worker stopped with exit code ${code}`)); + } }); - - const updatedSourceCode = removeIndexesFromSourceCode( - file.sourceCode, - indexesToRemove, - ); - - return { ...file, sourceCode: updatedSourceCode }; }); } - - run() { - console.info("\n"); - console.time("Splitting"); - - console.time("remove annotation from other groups"); - this.#removeAnnotationFromOtherGroups(); - console.timeEnd("remove annotation from other groups"); - - console.time("Get export map"); - const exportMap = this.#getExportMap(); - console.timeEnd("Get export map"); - - console.time("Remove invalid imports and usages"); - this.#removeInvalidImportsAndUsages(exportMap); - console.timeEnd("Remove invalid imports and usages"); - - console.time("Remove unused imports"); - this.#removeUnusedImports(); - console.timeEnd("Remove unused imports"); - - console.time("Remove unused files"); - this.#removeUnusedFiles(); - console.timeEnd("Remove unused files"); - - console.time("Remove unused exports"); - this.#removeUnusedExports(exportMap); - console.timeEnd("Remove unused exports"); - - console.time("Remove errors"); - this.#removeErrors(); - console.timeEnd("Remove errors"); - - console.timeEnd("Splitting"); - - return this.files; - } } export default SplitRunner; diff --git a/packages/cli/src/splitRunner/worker.ts b/packages/cli/src/splitRunner/worker.ts new file mode 100644 index 0000000..76d3ae3 --- /dev/null +++ b/packages/cli/src/splitRunner/worker.ts @@ -0,0 +1,188 @@ +import assert from "assert"; +import Parser from "tree-sitter"; +import { parentPort, workerData } from "worker_threads"; +import { Group } from "../dependencyManager/types"; +import { removeIndexesFromSourceCode } from "../helper/file"; +import { getLanguagePlugin } from "../languagesPlugins"; +import { DepExport } from "../languagesPlugins/types"; +import { File } from "./types"; + +const { + entrypointPath, + group, + files, +}: { + entrypointPath: string; + group: Group; + files: File[]; +} = workerData; + +let currentFiles = files; + +function removeAnnotationFromOtherGroups() { + currentFiles = currentFiles.map((file) => { + const languagePlugin = getLanguagePlugin(entrypointPath, file.path); + + const updatedSourceCode = languagePlugin.removeAnnotationFromOtherGroups( + file.sourceCode, + group + ); + return { ...file, sourceCode: updatedSourceCode }; + }); +} + +function getExportMap() { + const exportMap = new Map(); + + currentFiles.forEach((file) => { + const languagePlugin = getLanguagePlugin(entrypointPath, file.path); + const tree = languagePlugin.parser.parse(file.sourceCode); + const exports = languagePlugin.getExports(tree.rootNode); + exportMap.set(file.path, exports); + }); + + return exportMap; +} + +function removeInvalidImportsAndUsages(exportMap: Map) { + currentFiles = currentFiles.map((file) => { + const languagePlugin = getLanguagePlugin(entrypointPath, file.path); + + const updatedSourceCode = languagePlugin.cleanupInvalidImports( + file.path, + file.sourceCode, + exportMap + ); + + return { ...file, sourceCode: updatedSourceCode }; + }); +} + +function removeUnusedImports() { + currentFiles = currentFiles.map((file) => { + const languagePlugin = getLanguagePlugin(entrypointPath, file.path); + + const updatedSourceCode = languagePlugin.cleanupUnusedImports( + file.path, + file.sourceCode + ); + + return { ...file, sourceCode: updatedSourceCode }; + }); +} + +function removeUnusedFiles() { + let fileRemoved = true; + while (fileRemoved) { + fileRemoved = false; + + // We always want to keep the entrypoint file. + // It will never be imported anywhere, so we add it now. + const filesToKeep = new Set(); + filesToKeep.add(entrypointPath); + + currentFiles.forEach((file) => { + const languagePlugin = getLanguagePlugin(entrypointPath, file.path); + + const tree = languagePlugin.parser.parse(file.sourceCode); + + const imports = languagePlugin.getImports(file.path, tree.rootNode); + + imports.forEach((depImport) => { + if (depImport.isExternal || !depImport.source) { + // Ignore external dependencies + return; + } + + filesToKeep.add(depImport.source); + }); + }); + + const previousFilesLength = currentFiles.length; + + currentFiles = currentFiles.filter((file) => { + return filesToKeep.has(file.path); + }); + + if (currentFiles.length !== previousFilesLength) { + fileRemoved = true; + } + } +} + +function removeUnusedExports(exportMap: Map) { + let exportDeleted = true; + while (exportDeleted) { + exportDeleted = false; + + // TODO: Implement logic if needed. For now, we just assert exportMap. + assert(exportMap); + } + // TODO steps (left as comments): + // Step 1, create variable to track which export is used + // Step 2, iterate over all file imports. If the import is used, mark the export as used + // Step 3, iterate over each file, and remove the unused exports + // Repeat above step until no more unused exports are found +} + +function removeErrors() { + currentFiles = currentFiles.map((file) => { + const languagePlugin = getLanguagePlugin(entrypointPath, file.path); + + const tree = languagePlugin.parser.parse(file.sourceCode); + + const indexesToRemove: { startIndex: number; endIndex: number }[] = []; + + const query = new Parser.Query( + languagePlugin.parser.getLanguage(), + "(ERROR) @error" + ); + const errorCaptures = query.captures(tree.rootNode); + errorCaptures.forEach((capture) => { + indexesToRemove.push({ + startIndex: capture.node.startIndex, + endIndex: capture.node.endIndex, + }); + }); + + const updatedSourceCode = removeIndexesFromSourceCode( + file.sourceCode, + indexesToRemove + ); + + return { ...file, sourceCode: updatedSourceCode }; + }); +} + +(async () => { + console.time("remove annotation from other groups"); + removeAnnotationFromOtherGroups(); + console.timeEnd("remove annotation from other groups"); + + console.time("Get export map"); + const exportMap = getExportMap(); + console.timeEnd("Get export map"); + + console.time("Remove invalid imports and usages"); + removeInvalidImportsAndUsages(exportMap); + console.timeEnd("Remove invalid imports and usages"); + + console.time("Remove unused imports"); + removeUnusedImports(); + console.timeEnd("Remove unused imports"); + + console.time("Remove unused files"); + removeUnusedFiles(); + console.timeEnd("Remove unused files"); + + console.time("Remove unused exports"); + removeUnusedExports(exportMap); + console.timeEnd("Remove unused exports"); + + console.time("Remove errors"); + removeErrors(); + console.timeEnd("Remove errors"); + + // Send updated files back to the parent + parentPort?.postMessage(currentFiles); +})();