diff --git a/package-lock.json b/package-lock.json index ca8aa0b1..7d730409 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "unipept-desktop", - "version": "0.2.2", + "version": "0.3.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -17270,9 +17270,8 @@ "integrity": "sha512-g9AM+44twT+qQpg4kHhupamMoBr4QscjqjQY8Pdk7cdWHvZs5HuIVTNtH44IbAtN08x9+U3BTT+lp+ZEJpaoFw==" }, "unipept-web-components": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/unipept-web-components/-/unipept-web-components-1.2.2.tgz", - "integrity": "sha512-qQUG9M+QzB8ncNfABkHWvPJlXDJeXN3XldFCjjRZk4jIHEb2/PyLT0hwfWFGGZphV97dAZNX7CSz2heyO5fHkg==", + "version": "file:unipept-web-components-1.2.2.tgz", + "integrity": "sha512-4e1uQ8kFMns3mQiEjc4yG9tQfQZOL5HvUO9XyTsO511mLIPdfGcOnJfRvOwgwWGuj32OADaUJFyrRP56NZfI4w==", "requires": { "async": "^3.2.0", "axios": "^0.19.0", @@ -19382,23 +19381,33 @@ } }, "worker-loader": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-2.0.0.tgz", - "integrity": "sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.3.tgz", + "integrity": "sha512-yLUJqzloOnoh2/9OisTrUbUHd2a3Tfx8o8ilXHEQJ9Z/x/O/Ll+yZZOoVLT8G33IT2oCrjsIZ6jNB3OVIYCllA==", "dev": true, "requires": { - "loader-utils": "^1.0.0", - "schema-utils": "^0.4.0" + "loader-utils": "^2.0.0", + "schema-utils": "^2.7.0" }, "dependencies": { - "schema-utils": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", - "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", + "json5": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz", + "integrity": "sha512-KXPvOm8K9IJKFM0bmdn8QXh7udDh1g/giieX0NLCaMnb4hEiVFqnop2ImTXCc5e0/oHz3LTqmHGtExn5hfMkOA==", "dev": true, "requires": { - "ajv": "^6.1.0", - "ajv-keywords": "^3.1.0" + "minimist": "^1.2.5" + } + }, + "loader-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.0.tgz", + "integrity": "sha512-rP4F0h2RaWSvPEkD7BLDFQnvSf+nK+wr3ESUjNTyAGobqrijmW92zc+SO6d4p4B1wh7+B/Jg1mkQe5NYUEHtHQ==", + "dev": true, + "requires": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" } } } diff --git a/package.json b/package.json index 9045304f..1a0454e0 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "regenerator-runtime": "^0.13.3", "shared-memory-datastructures": "0.1.8", "threads": "^1.4.1", - "unipept-web-components": "1.2.2", + "unipept-web-components": "file:unipept-web-components-1.2.2.tgz", "uuid": "^7.0.3", "vue": "^2.6.12", "vue-class-component": "^7.1.0", @@ -92,7 +92,7 @@ "vue-cli-plugin-vuetify": "^0.6.3", "vue-template-compiler": "^2.6.12", "vuetify-loader": "^1.2.2", - "worker-loader": "^2.0.0" + "worker-loader": "^3.0.2" }, "eslintConfig": { "root": true, diff --git a/src/App.vue b/src/App.vue index 230ec88d..6791d674 100644 --- a/src/App.vue +++ b/src/App.vue @@ -71,6 +71,7 @@ import ConfigurationManager from "./logic/configuration/ConfigurationManager"; import Configuration from "./logic/configuration/Configuration"; import ErrorListener from "@/logic/filesystem/ErrorListener"; const electron = require("electron"); +import Worker from "worker-loader?inline=fallback!./logic/system/DesktopWorker.worker"; import { Assay, ProteomicsAssay, @@ -204,7 +205,7 @@ export default class App extends Vue implements ErrorListener { let config: Configuration = await configurationManager.readConfiguration(); NetworkConfiguration.BASE_URL = config.apiSource; NetworkConfiguration.PARALLEL_API_REQUESTS = config.maxParallelRequests; - QueueManager.initializeQueue(config.maxLongRunningTasks); + QueueManager.initializeQueue(config.maxLongRunningTasks, () => new Worker()); await this.$store.dispatch("setUseNativeTitlebar", config.useNativeTitlebar); } catch (err) { // TODO: show a proper error message to the user in case this happens diff --git a/src/background.ts b/src/background.ts index 296c3ac9..786968ea 100644 --- a/src/background.ts +++ b/src/background.ts @@ -8,7 +8,7 @@ import { autoUpdater } from "electron-updater"; import log from "electron-log"; import installExtension, { VUEJS_DEVTOOLS } from "electron-devtools-installer" -app.commandLine.appendSwitch("js-flags", "--max-old-space-size=4096"); +app.commandLine.appendSwitch("js-flags", "--max-old-space-size=4096 --expose_gc"); const isDevelopment = process.env.NODE_ENV !== "production" diff --git a/src/logic/communication/functional/CachedEcResponseCommunicator.ts b/src/logic/communication/functional/CachedEcResponseCommunicator.ts index 0aefecfe..234f03f5 100644 --- a/src/logic/communication/functional/CachedEcResponseCommunicator.ts +++ b/src/logic/communication/functional/CachedEcResponseCommunicator.ts @@ -1,5 +1,4 @@ -import { EcCode, EcResponseCommunicator, EcResponse } from "unipept-web-components"; -import { spawn, Worker } from "threads/dist"; +import { EcCode, EcResponseCommunicator, EcResponse, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; export default class CachedEcResponseCommunicator extends EcResponseCommunicator { @@ -29,17 +28,14 @@ export default class CachedEcResponseCommunicator extends EcResponseCommunicator await CachedEcResponseCommunicator.processing; } - if (!CachedEcResponseCommunicator.worker) { - CachedEcResponseCommunicator.worker = await spawn( - new Worker("./CachedEcResponseCommunicator.worker.ts") - ); - } - CachedEcResponseCommunicator.processing = CachedEcResponseCommunicator.worker.process( + CachedEcResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< + Map, [string, string, EcCode[], Map] + >("computeCachedEcResponses", [ __dirname, this.dbFile, codes, CachedEcResponseCommunicator.codeToResponses - ); + ]); CachedEcResponseCommunicator.codeToResponses = await CachedEcResponseCommunicator.processing; CachedEcResponseCommunicator.processing = undefined; diff --git a/src/logic/communication/functional/CachedEcResponseCommunicator.worker.ts b/src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts similarity index 50% rename from src/logic/communication/functional/CachedEcResponseCommunicator.worker.ts rename to src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts index f232aa35..f0bf0958 100644 --- a/src/logic/communication/functional/CachedEcResponseCommunicator.worker.ts +++ b/src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts @@ -1,16 +1,12 @@ -import { EcResponse } from "unipept-web-components/src/business/communication/functional/ec/EcResponse"; -import { EcCode } from "unipept-web-components/src/business/ontology/functional/ec/EcDefinition"; -import { expose } from "threads/worker"; +import { EcResponse, EcCode } from "unipept-web-components"; import Database from "better-sqlite3"; -expose({ process }) - -export default function process( - installationDir: string, - dbPath: string, - codes: EcCode[], - output: Map -): Map { +export async function compute([ + installationDir, + dbPath, + codes, + output +]: [string, string, EcCode[], Map]): Promise> { // @ts-ignore const db = new Database(dbPath, {}, installationDir); const stmt = db.prepare("SELECT * FROM ec_numbers WHERE `code` = ?"); diff --git a/src/logic/communication/functional/CachedGoResponseCommunicator.ts b/src/logic/communication/functional/CachedGoResponseCommunicator.ts index 0bb09c7f..23ecb99e 100644 --- a/src/logic/communication/functional/CachedGoResponseCommunicator.ts +++ b/src/logic/communication/functional/CachedGoResponseCommunicator.ts @@ -1,11 +1,9 @@ -import { GoResponse, GoResponseCommunicator, GoCode } from "unipept-web-components"; +import { GoResponse, GoResponseCommunicator, GoCode, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; -import { spawn, Worker } from "threads/dist"; export default class CachedGoResponseCommunicator extends GoResponseCommunicator { private static codeToResponses: Map = new Map(); private static processing: Promise>; - private static worker: any; private readonly dbFile: string; constructor() { @@ -29,17 +27,14 @@ export default class CachedGoResponseCommunicator extends GoResponseCommunicator await CachedGoResponseCommunicator.processing; } - if (!CachedGoResponseCommunicator.worker) { - CachedGoResponseCommunicator.worker = await spawn( - new Worker("./CachedGoResponseCommunicator.worker.ts") - ); - } - CachedGoResponseCommunicator.processing = CachedGoResponseCommunicator.worker.process( + CachedGoResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< + Map, [string, string, GoCode[], Map] + >("computeCachedGoResponses", [ __dirname, this.dbFile, codes, CachedGoResponseCommunicator.codeToResponses - ); + ]); CachedGoResponseCommunicator.codeToResponses = await CachedGoResponseCommunicator.processing; CachedGoResponseCommunicator.processing = undefined; diff --git a/src/logic/communication/functional/CachedGoResponseCommunicator.worker.ts b/src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts similarity index 74% rename from src/logic/communication/functional/CachedGoResponseCommunicator.worker.ts rename to src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts index 054b5931..cb5b1d88 100644 --- a/src/logic/communication/functional/CachedGoResponseCommunicator.worker.ts +++ b/src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts @@ -1,16 +1,11 @@ -import { expose } from "threads/worker"; import Database from "better-sqlite3"; import { GoResponse } from "unipept-web-components/src/business/communication/functional/go/GoResponse"; import { GoCode } from "unipept-web-components/src/business/ontology/functional/go/GoDefinition"; -expose({ process }) -export default function process( - installationDir: string, - dbPath: string, - codes: GoCode[], - output: Map -): Map { +export async function compute( + [installationDir, dbPath, codes, output]: [string, string, GoCode[], Map] +): Promise> { // @ts-ignore const db = new Database(dbPath, {}, installationDir); const stmt = db.prepare("SELECT * FROM go_terms WHERE `code` = ?"); diff --git a/src/logic/communication/functional/CachedInterproResponseCommunicator.ts b/src/logic/communication/functional/CachedInterproResponseCommunicator.ts index cee1c71b..6819ed77 100644 --- a/src/logic/communication/functional/CachedInterproResponseCommunicator.ts +++ b/src/logic/communication/functional/CachedInterproResponseCommunicator.ts @@ -1,6 +1,5 @@ -import { InterproCode, InterproResponse, InterproResponseCommunicator } from "unipept-web-components"; +import { InterproCode, InterproResponse, InterproResponseCommunicator, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; -import { spawn, Worker } from "threads/dist"; export default class CachedInterproResponseCommunicator extends InterproResponseCommunicator { private static codeToResponses: Map = new Map(); @@ -29,16 +28,16 @@ export default class CachedInterproResponseCommunicator extends InterproResponse await CachedInterproResponseCommunicator.processing; } - if (!CachedInterproResponseCommunicator.worker) { - CachedInterproResponseCommunicator.worker = await spawn( - new Worker("./CachedInterproResponseCommunicator.worker.ts") - ); - } - CachedInterproResponseCommunicator.processing = CachedInterproResponseCommunicator.worker.process( - __dirname, - this.dbFile, - codes, - CachedInterproResponseCommunicator.codeToResponses + CachedInterproResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< + Map, [string, string, InterproCode[], Map] + >( + "computeCachedInterproResponses", + [ + __dirname, + this.dbFile, + codes, + CachedInterproResponseCommunicator.codeToResponses + ] ); CachedInterproResponseCommunicator.codeToResponses = await CachedInterproResponseCommunicator.processing; diff --git a/src/logic/communication/functional/CachedInterproResponseCommunicator.worker.ts b/src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts similarity index 78% rename from src/logic/communication/functional/CachedInterproResponseCommunicator.worker.ts rename to src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts index 45df8d1b..07f001f4 100644 --- a/src/logic/communication/functional/CachedInterproResponseCommunicator.worker.ts +++ b/src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts @@ -1,18 +1,12 @@ -import { expose } from "threads/worker"; import Database from "better-sqlite3"; import InterproResponse from "unipept-web-components/src/business/communication/functional/interpro/InterproResponse"; import { InterproCode } from "unipept-web-components/src/business/ontology/functional/interpro/InterproDefinition"; import { EcResponse } from "unipept-web-components/src/business/communication/functional/ec/EcResponse"; import { EcCode } from "unipept-web-components/src/business/ontology/functional/ec/EcDefinition"; -expose({ process }) - -export default function process( - installationDir: string, - dbPath: string, - codes: InterproCode[], - output: Map -): Map { +export async function compute( + [installationDir, dbPath, codes, output]: [string, string, InterproCode[], Map] +): Promise> { // @ts-ignore const db = new Database(dbPath, {}, installationDir); const stmt = db.prepare("SELECT * FROM interpro_entries WHERE `code` = ?"); diff --git a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts index 1b102e8e..c160dace 100644 --- a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts +++ b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts @@ -1,7 +1,6 @@ -import { NcbiResponseCommunicator, NcbiId, NcbiResponse } from "unipept-web-components"; +import { NcbiResponseCommunicator, NcbiId, NcbiResponse, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; import { Database, Statement } from "better-sqlite3"; -import { spawn, Worker } from "threads/dist"; export default class CachedNcbiResponseCommunicator extends NcbiResponseCommunicator { private readonly database: Database; @@ -9,7 +8,6 @@ export default class CachedNcbiResponseCommunicator extends NcbiResponseCommunic private readonly extractStmt: Statement; private static codesProcessed: Map = new Map(); private static processing: Promise>; - private static worker: any; constructor() { super(); @@ -33,17 +31,19 @@ export default class CachedNcbiResponseCommunicator extends NcbiResponseCommunic const staticDatabaseManager = new StaticDatabaseManager(); - if (!CachedNcbiResponseCommunicator.worker) { - CachedNcbiResponseCommunicator.worker = await spawn( - new Worker("./CachedNcbiResponseCommunicator.worker.ts") - ); - } - CachedNcbiResponseCommunicator.processing = CachedNcbiResponseCommunicator.worker.process( - __dirname, - staticDatabaseManager.getDatabasePath(), - codes, - CachedNcbiResponseCommunicator.codesProcessed + CachedNcbiResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< + Map, + [string, string, NcbiId[], Map] + >( + "computeCachedNcbiResponses", + [ + __dirname, + staticDatabaseManager.getDatabasePath(), + codes, + CachedNcbiResponseCommunicator.codesProcessed + ] ); + CachedNcbiResponseCommunicator.codesProcessed = await CachedNcbiResponseCommunicator.processing; CachedNcbiResponseCommunicator.processing = undefined; } diff --git a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.worker.ts b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts similarity index 81% rename from src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.worker.ts rename to src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts index de4447a5..12d438c5 100644 --- a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.worker.ts +++ b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts @@ -1,12 +1,11 @@ import NcbiResponse from "unipept-web-components/src/business/communication/taxonomic/ncbi/NcbiResponse"; import { NcbiId } from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiTaxon"; import { NcbiRank } from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiRank"; -import { expose } from "threads/worker"; import Database from "better-sqlite3"; -expose({ process }) - -export default function process(installationDir: string, dbPath: string, ncbiIds: NcbiId[], output: Map): Map { +export async function compute( + [installationDir, dbPath, ncbiIds, output]: [string, string, NcbiId[], Map] +): Promise> { // @ts-ignore const database = new Database(dbPath, {}, installationDir); diff --git a/src/logic/filesystem/assay/AssayFileSystemDataReader.ts b/src/logic/filesystem/assay/AssayFileSystemDataReader.ts index d5848cd3..b989d01b 100644 --- a/src/logic/filesystem/assay/AssayFileSystemDataReader.ts +++ b/src/logic/filesystem/assay/AssayFileSystemDataReader.ts @@ -1,28 +1,21 @@ import FileSystemAssayVisitor from "@/logic/filesystem/assay/FileSystemAssayVisitor"; -import { ProteomicsAssay, IOException } from "unipept-web-components"; +import { ProteomicsAssay, IOException, QueueManager } from "unipept-web-components"; import { promises as fs } from "fs"; -import { spawn, Worker } from "threads" -import { Database } from "better-sqlite3"; export default class AssayFileSystemDataReader extends FileSystemAssayVisitor { - private static worker: any; - public async visitProteomicsAssay(mpAssay: ProteomicsAssay): Promise { const path: string = `${this.directoryPath}${mpAssay.getName()}.pep`; try { - const start = new Date().getTime(); const peptidesString: string = await fs.readFile(path, { encoding: "utf-8" }); - if (!AssayFileSystemDataReader.worker) { - AssayFileSystemDataReader.worker = await spawn(new Worker("./AssayFileSystemDataReader.worker.ts")); - } + const splitted = await QueueManager.getLongRunningQueue().pushTask( + "readAssay", + peptidesString + ); - const splitted = await AssayFileSystemDataReader.worker(peptidesString); - const end = new Date().getTime(); - console.log("Reading in peptides from file: " + (end - start) / 1000 + "s"); mpAssay.setPeptides(splitted); } catch (err) { // The file does not exist (yet), throw an exception diff --git a/src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts b/src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts similarity index 78% rename from src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts rename to src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts index 87c485d5..7e5c5f5f 100644 --- a/src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts +++ b/src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts @@ -1,9 +1,6 @@ -import { expose } from "threads/worker"; import { Peptide } from "unipept-web-components/src/business/ontology/raw/Peptide"; -expose(readAssay) - -async function readAssay(peptidesString: string): Promise { +export async function compute(peptidesString: string): Promise { const output = []; let terminatorPos = peptidesString.indexOf("\n"); let previousTerminatorPos = 0; diff --git a/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts b/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts index f2417649..33bc7b35 100644 --- a/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts +++ b/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts @@ -1,15 +1,13 @@ import FileSystemAssayVisitor from "./FileSystemAssayVisitor"; import { promises as fs } from "fs"; -import { ProteomicsAssay, IOException } from "unipept-web-components"; +import { ProteomicsAssay, IOException, QueueManager } from "unipept-web-components"; import SearchConfigFileSystemDestroyer from "@/logic/filesystem/configuration/SearchConfigFileSystemDestroyer"; -import { spawn, Worker } from "threads/dist"; import { Database } from "better-sqlite3"; /** * Removes both the metadata and raw data for an assay. */ export default class AssayFileSystemDestroyer extends FileSystemAssayVisitor { - private static worker: any; constructor( directoryPath: string, @@ -33,11 +31,10 @@ export default class AssayFileSystemDestroyer extends FileSystemAssayVisitor { // File does no longer exist, which is not an issue here. } - if (!AssayFileSystemDestroyer.worker) { - AssayFileSystemDestroyer.worker = await spawn(new Worker("./AssayFileSystemDestroyer.worker.ts")); - } - - await AssayFileSystemDestroyer.worker(assay.getId(), this.dbFile, __dirname); + await QueueManager.getLongRunningQueue().pushTask( + "destroyAssay", + [assay.getId(), this.dbFile, __dirname] + ); const configDestroyer = new SearchConfigFileSystemDestroyer(this.db); configDestroyer.visitSearchConfiguration(assay.getSearchConfiguration()); diff --git a/src/logic/filesystem/assay/AssayFileSystemDestroyer.worker.ts b/src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts similarity index 71% rename from src/logic/filesystem/assay/AssayFileSystemDestroyer.worker.ts rename to src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts index 6cf598c3..cde6e022 100644 --- a/src/logic/filesystem/assay/AssayFileSystemDestroyer.worker.ts +++ b/src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts @@ -1,9 +1,6 @@ -import { expose } from "threads/worker"; import Database from "better-sqlite3"; -expose(removeAssayFromDB) - -async function removeAssayFromDB(assayId: number, dbFile: string, installationDir: string): Promise { +export async function compute([assayId, dbFile, installationDir]: [string, string, string]): Promise { //@ts-ignore const db = new Database(dbFile, {}, installationDir); db.prepare("DELETE FROM pept2data WHERE `assay_id` = ?").run(assayId); diff --git a/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts b/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts index 5a1c63f5..97b830c1 100644 --- a/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts +++ b/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts @@ -1,20 +1,19 @@ - import { ProteomicsAssay, SearchConfiguration, PeptideTrust, PeptideData, - PeptideDataSerializer, NetworkConfiguration, DateUtils + PeptideDataSerializer, + NetworkConfiguration, + DateUtils, + QueueManager } from "unipept-web-components"; import ProcessedAssayResult from "@/logic/filesystem/assay/processed/ProcessedAssayResult"; import { Database } from "better-sqlite3"; import SearchConfigFileSystemReader from "@/logic/filesystem/configuration/SearchConfigFileSystemReader"; -import { spawn, TransferDescriptor, Worker } from "threads/dist"; import { ShareableMap } from "shared-memory-datastructures"; import SearchConfigFileSystemWriter from "@/logic/filesystem/configuration/SearchConfigFileSystemWriter"; -import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; -import MetadataCommunicator from "@/logic/communication/metadata/MetadataCommunicator"; export default class ProcessedAssayManager { private static worker: any; @@ -31,10 +30,6 @@ export default class ProcessedAssayManager { * @param assay The assay for which the deserialized results should be returned. */ public async readProcessingResults(assay: ProteomicsAssay): Promise { - if (!ProcessedAssayManager.worker) { - ProcessedAssayManager.worker = await spawn(new Worker("./ProcessedAssayManager.worker.ts")); - } - // Look up whether storage metadata with the given properties is present in the database. const row = this.db.prepare("SELECT * FROM storage_metadata WHERE assay_id = ?").get(assay.getId()); @@ -62,15 +57,22 @@ export default class ProcessedAssayManager { } // Now try to read the serialized pept2data from the database - const result: [ - TransferDescriptor, - TransferDescriptor, - PeptideTrust - ] | null = await ProcessedAssayManager.worker.readPept2Data( + const result = await QueueManager.getLongRunningQueue().pushTask< + [ + ArrayBuffer, + ArrayBuffer, + PeptideTrust + ] | null, + [ + string, + string, + string + ] + >("readPept2Data", [ __dirname, this.dbFile, assay.getId() - ) + ]); if (!result) { return null; @@ -79,8 +81,8 @@ export default class ProcessedAssayManager { const [indexBuffer, dataBuffer, trust] = result; const sharedMap = new ShareableMap(0, 0, new PeptideDataSerializer()); sharedMap.setBuffers( - indexBuffer.transferables[0] as SharedArrayBuffer, - dataBuffer.transferables[0] as SharedArrayBuffer + indexBuffer, + dataBuffer ); return new ProcessedAssayResult( @@ -103,14 +105,17 @@ export default class ProcessedAssayManager { // First try to write the pept2data information to the database. const buffers = pept2Data.getBuffers(); - await ProcessedAssayManager.worker.writePept2Data( + await QueueManager.getLongRunningQueue().pushTask< + void, + [string, ArrayBuffer, ArrayBuffer, PeptideTrust, string, string] + >("writePept2Data", [ __dirname, buffers[0], buffers[1], trust, assay.getId(), this.dbFile - ); + ]); // Now write the metadata to the database again. const existingConfig = assay.getSearchConfiguration(); diff --git a/src/logic/filesystem/assay/processed/ProcessedAssayManager.worker.ts b/src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts similarity index 69% rename from src/logic/filesystem/assay/processed/ProcessedAssayManager.worker.ts rename to src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts index 3bcbe62a..f5fd9a64 100644 --- a/src/logic/filesystem/assay/processed/ProcessedAssayManager.worker.ts +++ b/src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts @@ -1,17 +1,12 @@ -import { expose, TransferDescriptor } from "threads/dist"; import PeptideTrust from "unipept-web-components/src/business/processors/raw/PeptideTrust"; import Database from "better-sqlite3"; -import { Transfer } from "threads/worker"; -expose({ readPept2Data, writePept2Data }) -export function readPept2Data( - installationDir: string, - dbFile: string, - assayId: string -): [TransferDescriptor, TransferDescriptor, PeptideTrust] | null { +export async function readPept2Data( + [installationDir, dbFile, assayId]: [string, string, string] +): Promise<[ArrayBuffer, ArrayBuffer, PeptideTrust] | null> { // @ts-ignore - const db = new Database(dbFile, { timeout: 15000, verbose: console.warn }, installationDir); + const db = new Database(dbFile, { timeout: 15000 }, installationDir); const row = db.prepare("SELECT * FROM pept2data WHERE `assay_id` = ?").get(assayId); @@ -34,19 +29,21 @@ export function readPept2Data( trustRow.searched_peptides ); - return [Transfer(indexBuffer), Transfer(dataBuffer), peptideTrust] + return [indexBuffer, dataBuffer, peptideTrust] } -export function writePept2Data( - installationDir: string, - peptDataIndexBuffer: ArrayBuffer, - peptDataDataBuffer: ArrayBuffer, - peptideTrust: PeptideTrust, - assayId: string, - dbFile: string +export async function writePept2Data( + [ + installationDir, + peptDataIndexBuffer, + peptDataDataBuffer, + peptideTrust, + assayId, + dbFile + ]: [string, ArrayBuffer, ArrayBuffer, PeptideTrust, string, string] ) { //@ts-ignore - const db = new Database(dbFile, { timeout: 15000, verbose: console.warn }, installationDir); + const db = new Database(dbFile, { timeout: 15000 }, installationDir); // First delete all existing rows for this assay; db.prepare("DELETE FROM pept2data WHERE `assay_id` = ?").run(assayId); diff --git a/src/logic/system/DesktopWorker.worker.ts b/src/logic/system/DesktopWorker.worker.ts new file mode 100644 index 00000000..702bfec0 --- /dev/null +++ b/src/logic/system/DesktopWorker.worker.ts @@ -0,0 +1,37 @@ +import { createMessageEventListener, workerFunctionMap } from "unipept-web-components"; +import { + compute as computeCachedEcResponse +} from "@/logic/communication/functional/CachedEcResponseCommunicator.workerSource"; +import { + compute as computeCachedGoResponse +} from "@/logic/communication/functional/CachedGoResponseCommunicator.workerSource"; +import { + compute as computeCachedInterproResponse +} from "@/logic/communication/functional/CachedInterproResponseCommunicator.workerSource"; +import { + compute as computeCachedNcbiResponse +} from "@/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource"; +import { + compute as readAssay +} from "@/logic/filesystem/assay/AssayFileSystemDataReader.workerSource"; +import { + compute as destroyAssay +} from "@/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource"; +import { + readPept2Data, + writePept2Data +} from "@/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource"; + +workerFunctionMap.set("computeCachedEcResponses", computeCachedEcResponse); +workerFunctionMap.set("computeCachedGoResponses", computeCachedGoResponse); +workerFunctionMap.set("computeCachedInterproResponses", computeCachedInterproResponse); +workerFunctionMap.set("computeCachedNcbiResponses", computeCachedNcbiResponse); +workerFunctionMap.set("readAssay", readAssay); +workerFunctionMap.set("destroyAssay", destroyAssay); +workerFunctionMap.set("readPept2Data", readPept2Data); +workerFunctionMap.set("writePept2Data", writePept2Data); + +const ctx: Worker = self as any; + +// Respond to message from parent thread +ctx.addEventListener("message", createMessageEventListener(ctx)); diff --git a/src/typings/worker-loader.d.ts b/src/typings/worker-loader.d.ts index 10cba184..4ff0d016 100644 --- a/src/typings/worker-loader.d.ts +++ b/src/typings/worker-loader.d.ts @@ -1,7 +1,8 @@ -declare module "worker-loader!*" { +declare module "worker-loader?*" +{ class WebpackWorker extends Worker { constructor(); } - + export default WebpackWorker; } diff --git a/vue.config.js b/vue.config.js index b8ec961b..2161d2fa 100644 --- a/vue.config.js +++ b/vue.config.js @@ -37,9 +37,9 @@ module.exports = { }, configureWebpack:{ plugins: [ - new ThreadsPlugin({ - plugins: [new NodeTargetPlugin()] - }), + // new ThreadsPlugin({ + // plugins: [new NodeTargetPlugin()] + // }), new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1 })