Skip to content

Commit

Permalink
Initial setup for concurrent workers
Browse files Browse the repository at this point in the history
  • Loading branch information
@erbesharat
erbesharat committed Dec 8, 2024
1 parent 0cc7191 commit 20e9014
Show file tree
Hide file tree
Showing 6 changed files with 250 additions and 236 deletions.
2 changes: 1 addition & 1 deletion package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 19 additions & 12 deletions packages/cli/src/api/split.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import fs from "fs";
import path from "path";
import { z } from "zod";
import DependencyTreeManager from "../dependencyManager/dependencyManager";
import { Group } from "../dependencyManager/types";
import { cleanupOutputDir, createOutputDir } from "../helper/file";
import SplitRunner from "../splitRunner/splitRunner";
import { splitSchema } from "./helpers/validation";
import { z } from "zod";
import { Group } from "../dependencyManager/types";

export function split(payload: z.infer<typeof splitSchema>) {
console.time("split command");
const groupMap: Record<number, Group> = {};

// Get the dependency tree
const dependencyTreeManager = new DependencyTreeManager(
payload.entrypointPath,
payload.entrypointPath

Check failure on line 16 in packages/cli/src/api/split.ts

View workflow job for this annotation

GitHub Actions / lint 

Insert `,`
);

const outputDir = payload.outputDir || path.dirname(payload.entrypointPath);
Expand All @@ -33,15 +33,22 @@ export function split(payload: z.infer<typeof splitSchema>) {
const targetDir = path.dirname(payload.entrypointPath);
const annotationDirectory = path.join(outputDir, index.toString());

files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath,
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
files
.then((files) => {
files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath

Check failure on line 42 in packages/cli/src/api/split.ts

View workflow job for this annotation

GitHub Actions / lint 

Insert `,`
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
})
.catch((error) => {
console.error(error);
throw error;
});
});

// Store the processed annotations in the output directory
Expand Down
31 changes: 19 additions & 12 deletions packages/cli/src/commands/split.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import path from "path";
import fs from "fs";
import path from "path";
import DependencyTreeManager from "../dependencyManager/dependencyManager";
import { Group } from "../dependencyManager/types";
import { cleanupOutputDir, createOutputDir } from "../helper/file";
import SplitRunner from "../splitRunner/splitRunner";
import { Group } from "../dependencyManager/types";

export default function splitCommandHandler(
entrypointPath: string, // Path to the entrypoint file
outputDir: string, // Path to the output directory
outputDir: string // Path to the output directory

Check failure on line 10 in packages/cli/src/commands/split.ts

View workflow job for this annotation

GitHub Actions / lint 

Insert `,`
) {
const groupMap: Record<number, Group> = {};

Expand All @@ -27,15 +27,22 @@ export default function splitCommandHandler(
const targetDir = path.dirname(entrypointPath);
const annotationDirectory = path.join(outputDir, index.toString());

files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath,
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
files
.then((files) => {
files.forEach((file) => {
const relativeFileNamePath = path.relative(targetDir, file.path);
const destinationPath = path.join(
annotationDirectory,
relativeFileNamePath

Check failure on line 36 in packages/cli/src/commands/split.ts

View workflow job for this annotation

GitHub Actions / lint 

Insert `,`
);
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
fs.writeFileSync(destinationPath, file.sourceCode, "utf8");
});
})
.catch((error) => {
console.error(error);
throw error;
});
});

// Store the processed annotations in the output directory
Expand Down
6 changes: 3 additions & 3 deletions packages/cli/src/languagesPlugins/types.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Group } from "../dependencyManager/types";
import Parser from "tree-sitter";
import { Group } from "../dependencyManager/types";

export interface DepImportIdentifier {
// Specific to each programing languages. Used by the language plugins.
Expand Down Expand Up @@ -61,7 +61,7 @@ export interface LanguagePlugin {

removeAnnotationFromOtherGroups(
sourceCode: string,
groupToKeep: Group,
groupToKeep: Group

Check failure on line 64 in packages/cli/src/languagesPlugins/types.ts

View workflow job for this annotation

GitHub Actions / lint 

Insert `,`
): string;

getImports(filePath: string, node: Parser.SyntaxNode): DepImport[];
Expand All @@ -71,7 +71,7 @@ export interface LanguagePlugin {
cleanupInvalidImports(
filePath: string,
sourceCode: string,
exportMap: Map<string, DepExport[]>,
exportMap: Map<string, DepExport[]>

Check failure on line 74 in packages/cli/src/languagesPlugins/types.ts

View workflow job for this annotation

GitHub Actions / lint 

Insert `,`
): string;

cleanupUnusedImports(filePath: string, sourceCode: string): string;
Expand Down
228 changes: 20 additions & 208 deletions packages/cli/src/splitRunner/splitRunner.ts
View file
Original file line number Diff line number Diff line change
@@ -1,231 +1,43 @@
import { Group } from "../dependencyManager/types";
import { removeIndexesFromSourceCode } from "../helper/file";
import path from "path";
import { Worker } from "worker_threads";
import DependencyTreeManager from "../dependencyManager/dependencyManager";
import { Group } from "../dependencyManager/types";
import { File } from "./types";
import Parser from "tree-sitter";
import assert from "assert";
import { getLanguagePlugin } from "../languagesPlugins";
import { DepExport } from "../languagesPlugins/types";

class SplitRunner {
private dependencyTreeManager: DependencyTreeManager;
private entrypointPath: string;
private group: Group;
private files: File[];

constructor(dependencyTreeManager: DependencyTreeManager, group: Group) {
this.dependencyTreeManager = dependencyTreeManager;
this.entrypointPath = dependencyTreeManager.dependencyTree.path;
this.group = group;
this.files = dependencyTreeManager.getFiles();
}

#removeAnnotationFromOtherGroups() {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const updatedSourceCode = languagePlugin.removeAnnotationFromOtherGroups(
file.sourceCode,
this.group,
);
return { ...file, sourceCode: updatedSourceCode };
});
}

#getExportMap() {
const exportMap = new Map<string, DepExport[]>();

this.files.forEach((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const tree = languagePlugin.parser.parse(file.sourceCode);

const exports = languagePlugin.getExports(tree.rootNode);

exportMap.set(file.path, exports);
});

return exportMap;
}

#removeInvalidImportsAndUsages(exportMap: Map<string, DepExport[]>) {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const updatedSourceCode = languagePlugin.cleanupInvalidImports(
file.path,
file.sourceCode,
exportMap,
);

return { ...file, sourceCode: updatedSourceCode };
});
}

#removeUnusedImports() {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);
async run(): Promise<File[]> {
console.time("\nSplitting");

const updatedSourceCode = languagePlugin.cleanupUnusedImports(
file.path,
file.sourceCode,
);

return { ...file, sourceCode: updatedSourceCode };
const worker = new Worker(path.resolve(__dirname, "worker.js"), {
workerData: {
entrypointPath: this.dependencyTreeManager.dependencyTree.path,
group: this.group,
files: this.dependencyTreeManager.getFiles(),
},
});
}

#removeUnusedFiles() {
let fileRemoved = true;
while (fileRemoved) {
fileRemoved = false;

// We always want to keep the entrypoint file.
// It will never be imported anywhere, so we add it now.
const filesToKeep = new Set<string>();
filesToKeep.add(this.dependencyTreeManager.dependencyTree.path);

this.files.forEach((file) => {
const languagePlugin = getLanguagePlugin(
this.entrypointPath,
file.path,
);

const tree = languagePlugin.parser.parse(file.sourceCode);

const imports = languagePlugin.getImports(file.path, tree.rootNode);

imports.forEach((depImport) => {
if (depImport.isExternal || !depImport.source) {
// Ignore external dependencies
return;
}

filesToKeep.add(depImport.source);
});
});

const previousFilesLength = this.files.length;

this.files = this.files.filter((file) => {
return filesToKeep.has(file.path);
return new Promise<File[]>((resolve, reject) => {
worker.on("message", (updatedFiles: File[]) => {
console.timeEnd("Splitting");
resolve(updatedFiles);
});

if (this.files.length !== previousFilesLength) {
fileRemoved = true;
}
}
}

#removeUnusedExports(exportMap: Map<string, DepExport[]>) {
let exportDeleted = true;
while (exportDeleted) {
exportDeleted = false;

// const usedExportMap = new Map<string, Export>();

this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(
this.entrypointPath,
file.path,
);

const tree = languagePlugin.parser.parse(file.sourceCode);

const imports = languagePlugin.getImports(file.path, tree.rootNode);

imports.forEach((depImport) => {
if (depImport.isExternal || !depImport.source) {
// Ignore external dependencies
return;
}

// for each import, reconstruct the export map
const depExport = exportMap.get(depImport.source);
if (!depExport) {
throw new Error("Export not found");
}

// check named imports
});

return file;
});
}
// TODO
// Step 1, create variable to track which export is used
// Step 2, iterate over all file imports. If the import is used, mark the export as used
// Step 3, iterate over each file, and remove the unused exports

// Repeat above step until no more unused exports are found
assert(exportMap);
}

#removeErrors() {
this.files = this.files.map((file) => {
const languagePlugin = getLanguagePlugin(this.entrypointPath, file.path);

const tree = languagePlugin.parser.parse(file.sourceCode);

const indexesToRemove: { startIndex: number; endIndex: number }[] = [];

const query = new Parser.Query(
languagePlugin.parser.getLanguage(),
"(ERROR) @error",
);
const errorCaptures = query.captures(tree.rootNode);
errorCaptures.forEach((capture) => {
indexesToRemove.push({
startIndex: capture.node.startIndex,
endIndex: capture.node.endIndex,
});
worker.on("error", reject);
worker.on("exit", (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`));
}
});

const updatedSourceCode = removeIndexesFromSourceCode(
file.sourceCode,
indexesToRemove,
);

return { ...file, sourceCode: updatedSourceCode };
});
}

run() {
console.info("\n");
console.time("Splitting");

console.time("remove annotation from other groups");
this.#removeAnnotationFromOtherGroups();
console.timeEnd("remove annotation from other groups");

console.time("Get export map");
const exportMap = this.#getExportMap();
console.timeEnd("Get export map");

console.time("Remove invalid imports and usages");
this.#removeInvalidImportsAndUsages(exportMap);
console.timeEnd("Remove invalid imports and usages");

console.time("Remove unused imports");
this.#removeUnusedImports();
console.timeEnd("Remove unused imports");

console.time("Remove unused files");
this.#removeUnusedFiles();
console.timeEnd("Remove unused files");

console.time("Remove unused exports");
this.#removeUnusedExports(exportMap);
console.timeEnd("Remove unused exports");

console.time("Remove errors");
this.#removeErrors();
console.timeEnd("Remove errors");

console.timeEnd("Splitting");

return this.files;
}
}

export default SplitRunner;
Loading

0 comments on commit 20e9014

Please sign in to comment.