From 1ccc6c551aace54ce90a4a20c23cc53079fb7658 Mon Sep 17 00:00:00 2001 From: Pieter Verschaffelt Date: Mon, 5 Oct 2020 15:17:26 +0200 Subject: [PATCH] No more FS access in workers --- .github/workflows/publish.yml | 2 - package-lock.json | 34 ++--- package.json | 7 +- patches/better-sqlite3+7.0.1.patch | 57 -------- patches/bindings+1.5.0.patch | 67 ---------- src/App.vue | 3 +- src/background.ts | 2 +- .../navigation-drawers/AssayItem.vue | 5 +- .../navigation-drawers/StudyItem.vue | 8 +- .../communication/DesktopAssayProcessor.ts | 6 +- .../CachedEcResponseCommunicator.ts | 42 +++--- ...chedEcResponseCommunicator.workerSource.ts | 26 ---- .../CachedGoResponseCommunicator.ts | 44 +++---- ...chedGoResponseCommunicator.workerSource.ts | 26 ---- .../CachedInterproResponseCommunicator.ts | 49 ++++--- ...terproResponseCommunicator.workerSource.ts | 27 ---- .../static/StaticDatabaseManager.ts | 10 +- .../ncbi/CachedNcbiResponseCommunicator.ts | 68 +++++----- ...edNcbiResponseCommunicator.workerSource.ts | 30 ----- .../assay/AssayFileSystemDataReader.ts | 30 ++++- .../assay/AssayFileSystemDataReader.worker.ts | 30 +++++ .../AssayFileSystemDataReader.workerSource.ts | 13 -- .../assay/AssayFileSystemDestroyer.ts | 20 +-- .../AssayFileSystemDestroyer.workerSource.ts | 11 -- .../assay/AssayFileSystemMetaDataReader.ts | 30 +++-- .../assay/AssayFileSystemMetaDataWriter.ts | 45 ++++--- .../assay/FileSystemAssayChangeListener.ts | 8 +- .../assay/FileSystemAssayVisitor.ts | 9 +- .../assay/processed/ProcessedAssayManager.ts | 124 +++++++++++------- .../ProcessedAssayManager.workerSource.ts | 78 ----------- .../SearchConfigFileSystemDestroyer.ts | 13 +- .../SearchConfigFileSystemReader.ts | 8 +- .../SearchConfigFileSystemWriter.ts | 34 +++-- .../filesystem/database/DatabaseManager.ts | 32 +++++ .../filesystem/project/FileSystemWatcher.ts | 23 ++-- .../filesystem/project/ProjectManager.ts | 33 +++-- .../study/FileSystemStudyChangeListener.ts | 9 +- .../study/FileSystemStudyVisitor.ts | 5 +- .../study/StudyFileSystemDataReader.ts | 15 ++- .../study/StudyFileSystemMetaDataReader.ts | 5 +- .../study/StudyFileSystemMetaDataWriter.ts | 14 +- src/logic/system/DesktopWorker.worker.ts | 37 ------ src/main.ts | 2 +- src/state/PeptideSummaryStore.ts | 54 ++++++-- src/state/PeptideSummaryTable.worker.ts | 111 ++++++++-------- src/state/ProjectStore.ts | 24 ++-- 46 files changed, 548 insertions(+), 782 deletions(-) delete mode 100644 patches/better-sqlite3+7.0.1.patch delete mode 100644 patches/bindings+1.5.0.patch delete mode 100644 src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts delete mode 100644 src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts delete mode 100644 src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts delete mode 100644 src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts create mode 100644 src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts delete mode 100644 src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts delete mode 100644 src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts delete mode 100644 src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts create mode 100644 src/logic/filesystem/database/DatabaseManager.ts delete mode 100644 src/logic/system/DesktopWorker.worker.ts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7e4584fe..2c71412c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,8 +17,6 @@ jobs: - uses: actions/setup-node@v1 - name: Install dependencies run: npm install - - name: Patch dependencies - run: npx patch-package - name: Build application for ${{ matrix.os }} run: npm run electron:publish env: diff --git a/package-lock.json b/package-lock.json index 7d730409..b568bf91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "unipept-desktop", - "version": "0.3.1", + "version": "0.3.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -6998,21 +6998,21 @@ "integrity": "sha512-wFwTwCVebUrMgGeAwRL/NhZtHAUyT9n9yg4IMDwf10+6iCMxSkVq9MGCVEH+QZWo1nNidy8kNvwmv4zWHDTqvA==" }, "domhandler": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-3.2.0.tgz", - "integrity": "sha512-FnT5pxGpykNI10uuwyqae65Ysw7XBQJKDjDjlHgE/rsNtjr1FyGNVNQCVlM5hwcq9wkyWSqB+L5Z+Qa4khwLuA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-3.3.0.tgz", + "integrity": "sha512-J1C5rIANUbuYK+FuFL98650rihynUOEzRLxW+90bKZRWB6A1X1Tf82GxR1qAWLyfNPRvjqfip3Q5tdYlmAa9lA==", "requires": { "domelementtype": "^2.0.1" } }, "domutils": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.4.1.tgz", - "integrity": "sha512-AA5r2GD1Dljhxc+k4zD2HYQaDkDPBhTqmqF55wLNlxfhFQlqaYME8Jhmo2nKNBb+CNfPXE8SAjtF6SsZ0cza/w==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.4.2.tgz", + "integrity": "sha512-NKbgaM8ZJOecTZsIzW5gSuplsX2IWW2mIK7xVr8hTQF2v1CJWTmLZ1HOCh5sH+IzVPAGE5IucooOkvwBRAdowA==", "requires": { "dom-serializer": "^1.0.1", "domelementtype": "^2.0.1", - "domhandler": "^3.2.0" + "domhandler": "^3.3.0" } }, "dot-prop": { @@ -8790,9 +8790,9 @@ } }, "fork-ts-checker-webpack-plugin-v5": { - "version": "npm:fork-ts-checker-webpack-plugin@5.1.0", - "resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-5.1.0.tgz", - "integrity": "sha512-vuKyEjSLGbhQbEr5bifXXOkr9iV73L6n72mHoHIv7okvrf7O7z6RKeplM6C6ATPsukoQivij+Ba1vcptL60Z2g==", + "version": "npm:fork-ts-checker-webpack-plugin@5.2.0", + "resolved": "https://registry.npmjs.org/fork-ts-checker-webpack-plugin/-/fork-ts-checker-webpack-plugin-5.2.0.tgz", + "integrity": "sha512-NEKcI0+osT5bBFZ1SFGzJMQETjQWZrSvMO1g0nAR/w0t328Z41eN8BJEIZyFCl2HsuiJpa9AN474Nh2qLVwGLQ==", "dev": true, "optional": true, "requires": { @@ -8810,13 +8810,12 @@ }, "dependencies": { "ansi-styles": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", - "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, "optional": true, "requires": { - "@types/color-name": "^1.1.1", "color-convert": "^2.0.1" } }, @@ -17270,8 +17269,9 @@ "integrity": "sha512-g9AM+44twT+qQpg4kHhupamMoBr4QscjqjQY8Pdk7cdWHvZs5HuIVTNtH44IbAtN08x9+U3BTT+lp+ZEJpaoFw==" }, "unipept-web-components": { - "version": "file:unipept-web-components-1.2.2.tgz", - "integrity": "sha512-4e1uQ8kFMns3mQiEjc4yG9tQfQZOL5HvUO9XyTsO511mLIPdfGcOnJfRvOwgwWGuj32OADaUJFyrRP56NZfI4w==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/unipept-web-components/-/unipept-web-components-1.2.3.tgz", + "integrity": "sha512-8DpOwKkBAP62INjzWaG2RCyuo1LcjtZfD3N7CJg/8c/ixYHINr9xvAevH6WgTnLR8FD7B37IgRDzIHBD248AWQ==", "requires": { "async": "^3.2.0", "axios": "^0.19.0", diff --git a/package.json b/package.json index 1a0454e0..2751353e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "unipept-desktop", - "version": "0.3.1", + "version": "0.4.0", "private": true, "author": "Unipept Team (Ghent University)", "description": "A desktop equivalent of unipept.ugent.be. This app aims at high-throughput analysis of metaproteomics samples.", @@ -43,11 +43,9 @@ "follow-redirects": "^1.11.0", "jquery": "^3.4.1", "node-abi": "^2.18.0", - "patch-package": "^6.2.2", "regenerator-runtime": "^0.13.3", "shared-memory-datastructures": "0.1.8", - "threads": "^1.4.1", - "unipept-web-components": "file:unipept-web-components-1.2.2.tgz", + "unipept-web-components": "1.2.3", "uuid": "^7.0.3", "vue": "^2.6.12", "vue-class-component": "^7.1.0", @@ -85,7 +83,6 @@ "sass-loader": "^7.1.0", "speed-measure-webpack-plugin": "^1.3.3", "style-loader": "^1.0.0", - "threads-plugin": "^1.3.1", "typescript": "^3.9.5", "vue-cli-plugin-electron-builder": "^2.0.0-rc.4", "vue-cli-plugin-styleguidist": "^4.2.3", diff --git a/patches/better-sqlite3+7.0.1.patch b/patches/better-sqlite3+7.0.1.patch deleted file mode 100644 index 2e793efd..00000000 --- a/patches/better-sqlite3+7.0.1.patch +++ /dev/null @@ -1,57 +0,0 @@ -diff --git a/node_modules/better-sqlite3/lib/database.js b/node_modules/better-sqlite3/lib/database.js -index 19851cf..5b5cdea 100644 ---- a/node_modules/better-sqlite3/lib/database.js -+++ b/node_modules/better-sqlite3/lib/database.js -@@ -3,13 +3,33 @@ const fs = require('fs'); - const path = require('path'); - const util = require('./util'); - --const { -- Database: CPPDatabase, -- setErrorConstructor, --} = require('bindings')('better_sqlite3.node'); -+let CPPDatabase; -+let setErrorConstructor; - --function Database(filenameGiven, options) { -- if (filenameGiven == null) filenameGiven = ''; -+function initDatabase(installationDir) { -+ let required; -+ if (!installationDir || installationDir.includes("electron.asar")) { -+ required = require('bindings')('better_sqlite3.node'); -+ } else { -+ var requireFunc = typeof __webpack_require__ === 'function' ? __non_webpack_require__ : require; -+ required = requireFunc(path.join(installationDir, "node_modules/better-sqlite3/build/Release/better_sqlite3.node")); -+ } -+ -+ CPPDatabase = required.Database; -+ setErrorConstructor = required.setErrorConstructor; -+ setErrorConstructor(require('./sqlite-error')); -+ util.wrap(CPPDatabase, 'pragma', require('./pragma')); -+ util.wrap(CPPDatabase, 'function', require('./function')); -+ util.wrap(CPPDatabase, 'aggregate', require('./aggregate')); -+ util.wrap(CPPDatabase, 'backup', require('./backup')); -+ CPPDatabase.prototype.transaction = require('./transaction'); -+ CPPDatabase.prototype.constructor = Database; -+ Database.prototype = CPPDatabase.prototype; -+} -+ -+function Database(filenameGiven, options, installationDir) { -+ if (!CPPDatabase) initDatabase(installationDir); -+ if (filenameGiven == null) filenameGiven = ''; - if (options == null) options = {}; - if (typeof filenameGiven !== 'string') throw new TypeError('Expected first argument to be a string'); - if (typeof options !== 'object') throw new TypeError('Expected second argument to be an options object'); -@@ -34,12 +54,4 @@ function Database(filenameGiven, options) { - return new CPPDatabase(filename, filenameGiven, anonymous, readonly, fileMustExist, timeout, verbose || null); - } - --setErrorConstructor(require('./sqlite-error')); --util.wrap(CPPDatabase, 'pragma', require('./pragma')); --util.wrap(CPPDatabase, 'function', require('./function')); --util.wrap(CPPDatabase, 'aggregate', require('./aggregate')); --util.wrap(CPPDatabase, 'backup', require('./backup')); --CPPDatabase.prototype.transaction = require('./transaction'); --CPPDatabase.prototype.constructor = Database; --Database.prototype = CPPDatabase.prototype; - module.exports = Database; diff --git a/patches/bindings+1.5.0.patch b/patches/bindings+1.5.0.patch deleted file mode 100644 index a2ff48a6..00000000 --- a/patches/bindings+1.5.0.patch +++ /dev/null @@ -1,67 +0,0 @@ -diff --git a/node_modules/bindings/bindings.js b/node_modules/bindings/bindings.js -index 727413a..1fbb1ef 100644 ---- a/node_modules/bindings/bindings.js -+++ b/node_modules/bindings/bindings.js -@@ -79,7 +79,28 @@ function bindings(opts) { - - // Get the module root - if (!opts.module_root) { -- opts.module_root = exports.getRoot(exports.getFileName()); -+ const fileName = exports.getFileName(); -+ let module_root = exports.getRoot(fileName); -+ -+ // Filename is undefined when eval() was used to execute code with bindings in it. We don't have a valid -+ // module-root in that case and need to use a heuristic to hopefully find the correct directory. -+ if (!fileName) { -+ // Derive module_root from ".node"-filename -+ const possible_package_name = opts.bindings.replace('.node', ''); -+ let best_score = -1; -+ let best_match = ""; -+ for (const file of fs.readdirSync(join(module_root, 'node_modules'))) { -+ const current_score = compare(file, possible_package_name); -+ if (current_score > best_score) { -+ best_score = current_score; -+ best_match = file; -+ } -+ } -+ -+ module_root = join(module_root, 'node_modules', best_match); -+ } -+ -+ opts.module_root = module_root; - } - - // Ensure the given bindings name ends with .node -@@ -136,6 +157,21 @@ function bindings(opts) { - } - module.exports = exports = bindings; - -+/** -+ * Returns a similarity score for two strings. -+ * See: https://stackoverflow.com/questions/10473745/compare-strings-javascript-return-of-likely -+ */ -+function compare(strA, strB){ -+ for(var result = 0, i = strA.length; i--;){ -+ if(typeof strB[i] == 'undefined' || strA[i] == strB[i]); -+ else if(strA[i].toLowerCase() == strB[i].toLowerCase()) -+ result++; -+ else -+ result += 4; -+ } -+ return 1 - (result + 4*Math.abs(strA.length - strB.length))/(2*(strA.length+strB.length)); -+} -+ - /** - * Gets the filename of the JavaScript file that invokes this function. - * Used to help find the root directory of a module. -@@ -173,6 +209,10 @@ exports.getFileName = function getFileName(calling_file) { - Error.prepareStackTrace = origPST; - Error.stackTraceLimit = origSTL; - -+ if (!fileName) { -+ return ""; -+ } -+ - // handle filename that starts with "file://" - var fileSchema = 'file://'; - if (fileName.indexOf(fileSchema) === 0) { diff --git a/src/App.vue b/src/App.vue index 6791d674..230ec88d 100644 --- a/src/App.vue +++ b/src/App.vue @@ -71,7 +71,6 @@ import ConfigurationManager from "./logic/configuration/ConfigurationManager"; import Configuration from "./logic/configuration/Configuration"; import ErrorListener from "@/logic/filesystem/ErrorListener"; const electron = require("electron"); -import Worker from "worker-loader?inline=fallback!./logic/system/DesktopWorker.worker"; import { Assay, ProteomicsAssay, @@ -205,7 +204,7 @@ export default class App extends Vue implements ErrorListener { let config: Configuration = await configurationManager.readConfiguration(); NetworkConfiguration.BASE_URL = config.apiSource; NetworkConfiguration.PARALLEL_API_REQUESTS = config.maxParallelRequests; - QueueManager.initializeQueue(config.maxLongRunningTasks, () => new Worker()); + QueueManager.initializeQueue(config.maxLongRunningTasks); await this.$store.dispatch("setUseNativeTitlebar", config.useNativeTitlebar); } catch (err) { // TODO: show a proper error message to the user in case this happens diff --git a/src/background.ts b/src/background.ts index 786968ea..db866c69 100644 --- a/src/background.ts +++ b/src/background.ts @@ -27,7 +27,7 @@ async function createWindow() { let options = { width: 1200, height: 1000, - webPreferences: { nodeIntegration: true, nodeIntegrationInWorker: true, enableRemoteModule: true }, + webPreferences: { nodeIntegration: true, enableRemoteModule: true }, show: false }; diff --git a/src/components/navigation-drawers/AssayItem.vue b/src/components/navigation-drawers/AssayItem.vue index 970951ee..eef31319 100644 --- a/src/components/navigation-drawers/AssayItem.vue +++ b/src/components/navigation-drawers/AssayItem.vue @@ -356,7 +356,7 @@ export default class AssayItem extends Vue { newAssay.setSearchConfiguration(searchConfiguration); const metadataWriter = new AssayFileSystemMetaDataWriter( `${this.$store.getters.projectLocation}${this.study.getName()}`, - this.$store.getters.database, + this.$store.getters.dbManager, this.study ); await newAssay.accept(metadataWriter); @@ -371,8 +371,7 @@ export default class AssayItem extends Vue { this.removeConfirmationActive = false; const assayDestroyer = new AssayFileSystemDestroyer( `${this.$store.getters.projectLocation}${this.study.getName()}`, - this.$store.getters.database, - this.$store.getters.databaseFile + this.$store.getters.dbManager ); await this.assay.accept(assayDestroyer); } diff --git a/src/components/navigation-drawers/StudyItem.vue b/src/components/navigation-drawers/StudyItem.vue index eb20155e..525b82f8 100644 --- a/src/components/navigation-drawers/StudyItem.vue +++ b/src/components/navigation-drawers/StudyItem.vue @@ -246,7 +246,7 @@ export default class StudyItem extends Vue { // Completely destroy this study and wait for the file system watcher to pick the change up. const studyDestroyer = new StudyFileSystemRemover( `${this.$store.getters.projectLocation}${this.study.getName()}`, - this.$store.getters.database + this.$store.getters.dbManager ); await this.study.accept(studyDestroyer); } @@ -270,7 +270,7 @@ export default class StudyItem extends Vue { // Write metadata to disk const metaDataWriter = new AssayFileSystemMetaDataWriter( `${this.$store.getters.projectLocation}${this.study.getName()}`, - this.$store.getters.database, + this.$store.getters.dbManager, this.study ); @@ -301,7 +301,7 @@ export default class StudyItem extends Vue { // Write metadata to disk const metaDataWriter = new AssayFileSystemMetaDataWriter( `${this.$store.getters.projectLocation}${this.study.getName()}`, - this.$store.getters.database, + this.$store.getters.dbManager, this.study ); @@ -310,7 +310,7 @@ export default class StudyItem extends Vue { // Write the assay to disk. It will automatically be picked up by the file system watchers const assaySerializer = new AssayFileSystemDataWriter( `${this.$store.getters.projectLocation}${this.study.getName()}`, - this.$store.getters.database + this.$store.getters.dbManager ); await assay.accept(assaySerializer); diff --git a/src/logic/communication/DesktopAssayProcessor.ts b/src/logic/communication/DesktopAssayProcessor.ts index 392b2943..c17b327d 100644 --- a/src/logic/communication/DesktopAssayProcessor.ts +++ b/src/logic/communication/DesktopAssayProcessor.ts @@ -17,14 +17,14 @@ import ProcessedAssayManager from "@/logic/filesystem/assay/processed/ProcessedA import { Database } from "better-sqlite3"; import { ShareableMap } from "shared-memory-datastructures"; import MetadataCommunicator from "@/logic/communication/metadata/MetadataCommunicator"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; export default class DesktopAssayProcessor implements AssayProcessor { private pept2DataCommunicator: Pept2DataCommunicator; private cancelled: boolean = false; constructor( - private readonly db: Database, - private readonly dbFile: string, + private readonly dbManager: DatabaseManager, private readonly assay: ProteomicsAssay, private readonly progressListener?: ProgressListener ) {} @@ -62,7 +62,7 @@ export default class DesktopAssayProcessor implements AssayProcessor { peptideCountTable: CountTable, forceUpdate: boolean ): Promise<[ShareableMap, PeptideTrust]> { - const processedAssayManager = new ProcessedAssayManager(this.db, this.dbFile); + const processedAssayManager = new ProcessedAssayManager(this.dbManager); const processingResult = await processedAssayManager.readProcessingResults(this.assay); if (processingResult !== null && !forceUpdate) { diff --git a/src/logic/communication/functional/CachedEcResponseCommunicator.ts b/src/logic/communication/functional/CachedEcResponseCommunicator.ts index 234f03f5..e3832ecf 100644 --- a/src/logic/communication/functional/CachedEcResponseCommunicator.ts +++ b/src/logic/communication/functional/CachedEcResponseCommunicator.ts @@ -1,48 +1,46 @@ import { EcCode, EcResponseCommunicator, EcResponse, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; +import { Database } from "better-sqlite3"; export default class CachedEcResponseCommunicator extends EcResponseCommunicator { private static codeToResponses: Map = new Map(); private static processing: Promise>; - private static worker: any; - private readonly dbFile: string; + private readonly dbManager: DatabaseManager; constructor() { super(); try { const staticDatabaseManager = new StaticDatabaseManager(); - staticDatabaseManager.getDatabase(); - this.dbFile = staticDatabaseManager.getDatabasePath(); + this.dbManager = staticDatabaseManager.getDatabaseManager(); } catch (err) { console.warn("Gracefully falling back to online communicators..."); - this.dbFile = ""; } } public async process(codes: EcCode[]): Promise { - if (!this.dbFile) { + if (!this.dbManager) { return super.process(codes); } - while (CachedEcResponseCommunicator.processing) { - await CachedEcResponseCommunicator.processing; - } + await this.dbManager.performQuery((db: Database) => { + const stmt = db.prepare("SELECT * FROM ec_numbers WHERE `code` = ?"); + + for (const code of codes) { + const row = stmt.get(code.substr(3)); - CachedEcResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< - Map, [string, string, EcCode[], Map] - >("computeCachedEcResponses", [ - __dirname, - this.dbFile, - codes, - CachedEcResponseCommunicator.codeToResponses - ]); - - CachedEcResponseCommunicator.codeToResponses = await CachedEcResponseCommunicator.processing; - CachedEcResponseCommunicator.processing = undefined; + if (row) { + CachedEcResponseCommunicator.codeToResponses.set(code, { + code: "EC:" + row.code, + name: row.name + }); + } + } + }); } public getResponse(code: EcCode): EcResponse { - if (!this.dbFile) { + if (!this.dbManager) { return super.getResponse(code); } @@ -50,7 +48,7 @@ export default class CachedEcResponseCommunicator extends EcResponseCommunicator } public getResponseMap(): Map { - if (!this.dbFile) { + if (!this.dbManager) { return super.getResponseMap(); } diff --git a/src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts b/src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts deleted file mode 100644 index f0bf0958..00000000 --- a/src/logic/communication/functional/CachedEcResponseCommunicator.workerSource.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { EcResponse, EcCode } from "unipept-web-components"; -import Database from "better-sqlite3"; - -export async function compute([ - installationDir, - dbPath, - codes, - output -]: [string, string, EcCode[], Map]): Promise> { - // @ts-ignore - const db = new Database(dbPath, {}, installationDir); - const stmt = db.prepare("SELECT * FROM ec_numbers WHERE `code` = ?"); - - for (const code of codes) { - const row = stmt.get(code.substr(3)); - - if (row) { - output.set(code, { - code: "EC:" + row.code, - name: row.name - }); - } - } - - return output; -} diff --git a/src/logic/communication/functional/CachedGoResponseCommunicator.ts b/src/logic/communication/functional/CachedGoResponseCommunicator.ts index 23ecb99e..e8369044 100644 --- a/src/logic/communication/functional/CachedGoResponseCommunicator.ts +++ b/src/logic/communication/functional/CachedGoResponseCommunicator.ts @@ -1,47 +1,47 @@ import { GoResponse, GoResponseCommunicator, GoCode, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; +import { Database } from "better-sqlite3"; export default class CachedGoResponseCommunicator extends GoResponseCommunicator { private static codeToResponses: Map = new Map(); private static processing: Promise>; - private readonly dbFile: string; + private readonly dbManager: DatabaseManager; constructor() { super(); try { const staticDatabaseManager = new StaticDatabaseManager(); - staticDatabaseManager.getDatabase(); - this.dbFile = staticDatabaseManager.getDatabasePath(); + this.dbManager = staticDatabaseManager.getDatabaseManager(); } catch (err) { console.warn("Gracefully falling back to online communicators..."); - this.dbFile = ""; } } public async process(codes: string[]): Promise { - if (!this.dbFile) { + if (!this.dbManager) { return super.process(codes); } - while (CachedGoResponseCommunicator.processing) { - await CachedGoResponseCommunicator.processing; - } - - CachedGoResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< - Map, [string, string, GoCode[], Map] - >("computeCachedGoResponses", [ - __dirname, - this.dbFile, - codes, - CachedGoResponseCommunicator.codeToResponses - ]); - - CachedGoResponseCommunicator.codeToResponses = await CachedGoResponseCommunicator.processing; - CachedGoResponseCommunicator.processing = undefined; + await this.dbManager.performQuery((db: Database) => { + const stmt = db.prepare("SELECT * FROM go_terms WHERE `code` = ?"); + + for (const code of codes) { + const row = stmt.get(code); + + if (row) { + CachedGoResponseCommunicator.codeToResponses.set(code, { + code: row.code, + namespace: row.namespace, + name: row.name + }); + } + } + }); } public getResponse(code: string): GoResponse { - if (!this.dbFile) { + if (!this.dbManager) { return super.getResponse(code); } @@ -49,7 +49,7 @@ export default class CachedGoResponseCommunicator extends GoResponseCommunicator } public getResponseMap(): Map { - if (!this.dbFile) { + if (!this.dbManager) { return super.getResponseMap(); } diff --git a/src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts b/src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts deleted file mode 100644 index cb5b1d88..00000000 --- a/src/logic/communication/functional/CachedGoResponseCommunicator.workerSource.ts +++ /dev/null @@ -1,26 +0,0 @@ -import Database from "better-sqlite3"; -import { GoResponse } from "unipept-web-components/src/business/communication/functional/go/GoResponse"; -import { GoCode } from "unipept-web-components/src/business/ontology/functional/go/GoDefinition"; - - -export async function compute( - [installationDir, dbPath, codes, output]: [string, string, GoCode[], Map] -): Promise> { - // @ts-ignore - const db = new Database(dbPath, {}, installationDir); - const stmt = db.prepare("SELECT * FROM go_terms WHERE `code` = ?"); - - for (const code of codes) { - const row = stmt.get(code); - - if (row) { - output.set(code, { - code: row.code, - name: row.name, - namespace: row.namespace - }); - } - } - - return output; -} diff --git a/src/logic/communication/functional/CachedInterproResponseCommunicator.ts b/src/logic/communication/functional/CachedInterproResponseCommunicator.ts index 6819ed77..3a5cf384 100644 --- a/src/logic/communication/functional/CachedInterproResponseCommunicator.ts +++ b/src/logic/communication/functional/CachedInterproResponseCommunicator.ts @@ -1,51 +1,46 @@ import { InterproCode, InterproResponse, InterproResponseCommunicator, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; +import { Database } from "better-sqlite3"; export default class CachedInterproResponseCommunicator extends InterproResponseCommunicator { private static codeToResponses: Map = new Map(); - private static processing: Promise>; - private static worker: any; - private readonly dbFile: string; + private readonly dbManager: DatabaseManager; constructor() { super(); try { const staticDatabaseManager = new StaticDatabaseManager(); - staticDatabaseManager.getDatabase(); - this.dbFile = staticDatabaseManager.getDatabasePath(); + this.dbManager = staticDatabaseManager.getDatabaseManager(); } catch (err) { console.warn("Gracefully falling back to online communicators..."); - this.dbFile = ""; } } public async process(codes: string[]): Promise { - if (!this.dbFile) { + if (!this.dbManager) { return super.process(codes as unknown as string[]); } - while (CachedInterproResponseCommunicator.processing) { - await CachedInterproResponseCommunicator.processing; - } - - CachedInterproResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< - Map, [string, string, InterproCode[], Map] - >( - "computeCachedInterproResponses", - [ - __dirname, - this.dbFile, - codes, - CachedInterproResponseCommunicator.codeToResponses - ] - ); - - CachedInterproResponseCommunicator.codeToResponses = await CachedInterproResponseCommunicator.processing; - CachedInterproResponseCommunicator.processing = undefined; + await this.dbManager.performQuery((db: Database) => { + const stmt = db.prepare("SELECT * FROM interpro_entries WHERE `code` = ?"); + + for (const code of codes) { + const row = stmt.get(code.substr(4)); + + if (row) { + CachedInterproResponseCommunicator.codeToResponses.set(code, { + code: "IPR:" + row.code, + name: row.name, + category: row.category + }); + } + } + }); } public getResponse(code: InterproCode): InterproResponse { - if (!this.dbFile) { + if (!this.dbManager) { return super.getResponse(code as unknown as string); } @@ -53,7 +48,7 @@ export default class CachedInterproResponseCommunicator extends InterproResponse } public getResponseMap(): Map { - if (!this.dbFile) { + if (!this.dbManager) { return super.getResponseMap(); } diff --git a/src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts b/src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts deleted file mode 100644 index 07f001f4..00000000 --- a/src/logic/communication/functional/CachedInterproResponseCommunicator.workerSource.ts +++ /dev/null @@ -1,27 +0,0 @@ -import Database from "better-sqlite3"; -import InterproResponse from "unipept-web-components/src/business/communication/functional/interpro/InterproResponse"; -import { InterproCode } from "unipept-web-components/src/business/ontology/functional/interpro/InterproDefinition"; -import { EcResponse } from "unipept-web-components/src/business/communication/functional/ec/EcResponse"; -import { EcCode } from "unipept-web-components/src/business/ontology/functional/ec/EcDefinition"; - -export async function compute( - [installationDir, dbPath, codes, output]: [string, string, InterproCode[], Map] -): Promise> { - // @ts-ignore - const db = new Database(dbPath, {}, installationDir); - const stmt = db.prepare("SELECT * FROM interpro_entries WHERE `code` = ?"); - - for (const code of codes) { - const row = stmt.get(code.substr(4)); - - if (row) { - output.set(code, { - code: "IPR:" + row.code, - name: row.name, - category: row.category - }); - } - } - - return output; -} diff --git a/src/logic/communication/static/StaticDatabaseManager.ts b/src/logic/communication/static/StaticDatabaseManager.ts index 816f8de6..e409c0c5 100644 --- a/src/logic/communication/static/StaticDatabaseManager.ts +++ b/src/logic/communication/static/StaticDatabaseManager.ts @@ -6,6 +6,7 @@ import { ProgressListener } from "unipept-web-components"; import yauzl, { Entry, ZipFile } from "yauzl"; import Database, { Database as DatabaseType } from "better-sqlite3"; import { Readable } from "stream"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; const { app } = require("electron").remote; @@ -22,6 +23,8 @@ export default class StaticDatabaseManager { public static readonly STATIC_DB_VERSION_POINTER = "https://raw.githubusercontent.com/unipept/make-database/master/workflows/static_database/version.txt"; public static readonly STATIC_DB_URL = "https://github.com/unipept/make-database/releases/latest/download/"; + private static dbManager: DatabaseManager; + /** * Check if the most recent version of the static information database is present in the userdata folder. If the * database is not present or if it's outdated this function returns true. @@ -172,8 +175,11 @@ export default class StaticDatabaseManager { * Returns an instance of the static database that can be queried immediately. Throws an error if the static * database file seems not to be present. */ - public getDatabase(): DatabaseType { - return new Database(this.getDatabasePath(), { fileMustExist: true }); + public getDatabaseManager(): DatabaseManager { + if (!StaticDatabaseManager.dbManager) { + StaticDatabaseManager.dbManager = new DatabaseManager(this.getDatabasePath()); + } + return StaticDatabaseManager.dbManager; } public getDatabasePath(): string { diff --git a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts index c160dace..519b2ab0 100644 --- a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts +++ b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.ts @@ -1,63 +1,59 @@ import { NcbiResponseCommunicator, NcbiId, NcbiResponse, QueueManager } from "unipept-web-components"; import StaticDatabaseManager from "@/logic/communication/static/StaticDatabaseManager"; -import { Database, Statement } from "better-sqlite3"; +import { Database } from "better-sqlite3"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; +import { NcbiRank } from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiRank"; export default class CachedNcbiResponseCommunicator extends NcbiResponseCommunicator { - private readonly database: Database; - private fallbackCommunicator: NcbiResponseCommunicator; - private readonly extractStmt: Statement; + private readonly dbManager: DatabaseManager; private static codesProcessed: Map = new Map(); - private static processing: Promise>; constructor() { super(); - const staticDatabaseManager = new StaticDatabaseManager(); try { - this.database = staticDatabaseManager.getDatabase(); + const staticDatabaseManager = new StaticDatabaseManager(); + this.dbManager = staticDatabaseManager.getDatabaseManager(); } catch (err) { console.warn("Gracefully falling back to online communicators..."); - this.database = null; - this.fallbackCommunicator = new NcbiResponseCommunicator(); } } public async process(codes: NcbiId[]): Promise { - if (!this.database) { - await this.fallbackCommunicator.process(codes); + if (!this.dbManager) { + await super.process(codes); } else { - while (CachedNcbiResponseCommunicator.processing) { - await CachedNcbiResponseCommunicator.processing; - } - - const staticDatabaseManager = new StaticDatabaseManager(); - - CachedNcbiResponseCommunicator.processing = QueueManager.getLongRunningQueue().pushTask< - Map, - [string, string, NcbiId[], Map] - >( - "computeCachedNcbiResponses", - [ - __dirname, - staticDatabaseManager.getDatabasePath(), - codes, - CachedNcbiResponseCommunicator.codesProcessed - ] - ); - - CachedNcbiResponseCommunicator.codesProcessed = await CachedNcbiResponseCommunicator.processing; - CachedNcbiResponseCommunicator.processing = undefined; + await this.dbManager.performQuery((db: Database) => { + const extractStmt = db.prepare( + "SELECT * FROM taxons INNER JOIN lineages ON taxons.id = lineages.taxon_id WHERE `id` = ?" + ); + + for (const id of codes) { + const row = extractStmt.get(id); + if (row) { + const lineage = Object.values(NcbiRank).map(rank => row[rank]).map(el => el === "\\N" ? null : el); + CachedNcbiResponseCommunicator.codesProcessed.set(id, { + id: row.id, + name: row.name, + rank: row.rank, + lineage: lineage + }); + } + } + }) } } public getResponse(id: NcbiId): NcbiResponse { - if (!this.database) { - return this.fallbackCommunicator.getResponse(id); - } else { - return CachedNcbiResponseCommunicator.codesProcessed.get(id); + if (!this.dbManager) { + return super.getResponse(id); } + return CachedNcbiResponseCommunicator.codesProcessed.get(id); } public getResponseMap(): Map { + if (!this.dbManager) { + return super.getResponseMap(); + } return CachedNcbiResponseCommunicator.codesProcessed; } } diff --git a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts b/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts deleted file mode 100644 index 12d438c5..00000000 --- a/src/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource.ts +++ /dev/null @@ -1,30 +0,0 @@ -import NcbiResponse from "unipept-web-components/src/business/communication/taxonomic/ncbi/NcbiResponse"; -import { NcbiId } from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiTaxon"; -import { NcbiRank } from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiRank"; -import Database from "better-sqlite3"; - -export async function compute( - [installationDir, dbPath, ncbiIds, output]: [string, string, NcbiId[], Map] -): Promise> { - // @ts-ignore - const database = new Database(dbPath, {}, installationDir); - - const extractStmt = database.prepare( - "SELECT * FROM taxons INNER JOIN lineages ON taxons.id = lineages.taxon_id WHERE `id` = ?" - ); - - for (const id of ncbiIds) { - const row = extractStmt.get(id); - if (row) { - const lineage = Object.values(NcbiRank).map(rank => row[rank]).map(el => el === "\\N" ? null : el); - output.set(id, { - id: row.id, - name: row.name, - rank: row.rank, - lineage: lineage - }); - } - } - - return output; -} diff --git a/src/logic/filesystem/assay/AssayFileSystemDataReader.ts b/src/logic/filesystem/assay/AssayFileSystemDataReader.ts index b989d01b..b11271ef 100644 --- a/src/logic/filesystem/assay/AssayFileSystemDataReader.ts +++ b/src/logic/filesystem/assay/AssayFileSystemDataReader.ts @@ -1,8 +1,12 @@ import FileSystemAssayVisitor from "@/logic/filesystem/assay/FileSystemAssayVisitor"; import { ProteomicsAssay, IOException, QueueManager } from "unipept-web-components"; import { promises as fs } from "fs"; +import Worker from "worker-loader?inline=fallback!./AssayFileSystemDataReader.worker"; export default class AssayFileSystemDataReader extends FileSystemAssayVisitor { + private static inProgress: Promise; + private static worker: Worker; + public async visitProteomicsAssay(mpAssay: ProteomicsAssay): Promise { const path: string = `${this.directoryPath}${mpAssay.getName()}.pep`; @@ -11,15 +15,33 @@ export default class AssayFileSystemDataReader extends FileSystemAssayVisitor { encoding: "utf-8" }); - const splitted = await QueueManager.getLongRunningQueue().pushTask( - "readAssay", - peptidesString - ); + while (AssayFileSystemDataReader.inProgress) { + await AssayFileSystemDataReader.inProgress; + } + + AssayFileSystemDataReader.inProgress = new Promise((resolve) => { + if (!AssayFileSystemDataReader.worker) { + AssayFileSystemDataReader.worker = new Worker(); + } + + const eventListener = (message: MessageEvent) => { + AssayFileSystemDataReader.worker.removeEventListener("message", eventListener); + resolve(message.data); + }; + + AssayFileSystemDataReader.worker.addEventListener("message", eventListener); + + AssayFileSystemDataReader.worker.postMessage(peptidesString); + }); + + const splitted = await AssayFileSystemDataReader.inProgress; + AssayFileSystemDataReader.inProgress = undefined; mpAssay.setPeptides(splitted); } catch (err) { // The file does not exist (yet), throw an exception throw new IOException(err); } + } } diff --git a/src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts b/src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts new file mode 100644 index 00000000..82953cb1 --- /dev/null +++ b/src/logic/filesystem/assay/AssayFileSystemDataReader.worker.ts @@ -0,0 +1,30 @@ +import { Peptide } from "unipept-web-components/src/business/ontology/raw/Peptide"; + +const ctx: Worker = self as any; + +// Respond to message from parent thread +ctx.addEventListener("message", (message: MessageEvent) => { + ctx.postMessage(compute(message.data)); + + try { + // This is unfortunately required to get the workers to stop consuming 100% CPU once they're done + // processing... + if (global && global.gc) { + global.gc(); + } + } catch (err) { + // GC is not available. + } +}); + +export function compute(peptidesString: string): Peptide[] { + const output = []; + let terminatorPos = peptidesString.indexOf("\n"); + let previousTerminatorPos = 0; + while (terminatorPos !== -1) { + output.push(peptidesString.substring(previousTerminatorPos, terminatorPos).trimEnd()); + previousTerminatorPos = terminatorPos + 1; + terminatorPos = peptidesString.indexOf("\n", previousTerminatorPos); + } + return output; +} diff --git a/src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts b/src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts deleted file mode 100644 index 7e5c5f5f..00000000 --- a/src/logic/filesystem/assay/AssayFileSystemDataReader.workerSource.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Peptide } from "unipept-web-components/src/business/ontology/raw/Peptide"; - -export async function compute(peptidesString: string): Promise { - const output = []; - let terminatorPos = peptidesString.indexOf("\n"); - let previousTerminatorPos = 0; - while (terminatorPos !== -1) { - output.push(peptidesString.substring(previousTerminatorPos, terminatorPos).trimEnd()); - previousTerminatorPos = terminatorPos + 1; - terminatorPos = peptidesString.indexOf("\n", previousTerminatorPos); - } - return output; -} diff --git a/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts b/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts index 33bc7b35..0a5e9256 100644 --- a/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts +++ b/src/logic/filesystem/assay/AssayFileSystemDestroyer.ts @@ -3,6 +3,7 @@ import { promises as fs } from "fs"; import { ProteomicsAssay, IOException, QueueManager } from "unipept-web-components"; import SearchConfigFileSystemDestroyer from "@/logic/filesystem/configuration/SearchConfigFileSystemDestroyer"; import { Database } from "better-sqlite3"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; /** * Removes both the metadata and raw data for an assay. @@ -11,10 +12,9 @@ export default class AssayFileSystemDestroyer extends FileSystemAssayVisitor { constructor( directoryPath: string, - db: Database, - private readonly dbFile: string + dbManager: DatabaseManager, ) { - super(directoryPath, db); + super(directoryPath, dbManager); } @@ -31,12 +31,16 @@ export default class AssayFileSystemDestroyer extends FileSystemAssayVisitor { // File does no longer exist, which is not an issue here. } - await QueueManager.getLongRunningQueue().pushTask( - "destroyAssay", - [assay.getId(), this.dbFile, __dirname] - ); + const assayId = assay.getId(); - const configDestroyer = new SearchConfigFileSystemDestroyer(this.db); + await this.dbManager.performQuery((db: Database) => { + db.prepare("DELETE FROM pept2data WHERE `assay_id` = ?").run(assayId); + db.prepare("DELETE FROM peptide_trust WHERE `assay_id` = ?").run(assayId); + db.prepare("DELETE FROM storage_metadata WHERE `assay_id` = ?").run(assayId); + db.prepare("DELETE FROM assays WHERE `id` = ?").run(assayId); + }); + + const configDestroyer = new SearchConfigFileSystemDestroyer(this.dbManager); configDestroyer.visitSearchConfiguration(assay.getSearchConfiguration()); } catch (e) { throw new IOException(e); diff --git a/src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts b/src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts deleted file mode 100644 index cde6e022..00000000 --- a/src/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource.ts +++ /dev/null @@ -1,11 +0,0 @@ -import Database from "better-sqlite3"; - -export async function compute([assayId, dbFile, installationDir]: [string, string, string]): Promise { - //@ts-ignore - const db = new Database(dbFile, {}, installationDir); - db.prepare("DELETE FROM pept2data WHERE `assay_id` = ?").run(assayId); - db.prepare("DELETE FROM peptide_trust WHERE `assay_id` = ?").run(assayId); - db.prepare("DELETE FROM storage_metadata WHERE `assay_id` = ?").run(assayId); - db.prepare("DELETE FROM assays WHERE `id` = ?").run(assayId); - db.close(); -} diff --git a/src/logic/filesystem/assay/AssayFileSystemMetaDataReader.ts b/src/logic/filesystem/assay/AssayFileSystemMetaDataReader.ts index f18480e8..84b6cb34 100644 --- a/src/logic/filesystem/assay/AssayFileSystemMetaDataReader.ts +++ b/src/logic/filesystem/assay/AssayFileSystemMetaDataReader.ts @@ -2,34 +2,40 @@ import { Database, Statement } from "better-sqlite3"; import FileSystemAssayVisitor from "@/logic/filesystem/assay/FileSystemAssayVisitor"; import { ProteomicsAssay, SearchConfiguration, Study } from "unipept-web-components"; import SearchConfigFileSystemReader from "@/logic/filesystem/configuration/SearchConfigFileSystemReader"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; export default class AssayFileSystemMetaDataReader extends FileSystemAssayVisitor { constructor( directoryPath: string, - db: Database, + dbManager: DatabaseManager, private readonly study?: Study ) { - super(directoryPath, db); + super(directoryPath, dbManager); } public async visitProteomicsAssay(mpAssay: ProteomicsAssay): Promise { - let row = this.db.prepare("SELECT * FROM assays WHERE assays.id = ?").get(mpAssay.getId()); + let row = await this.dbManager.performQuery( + (db: Database) => db.prepare("SELECT * FROM assays WHERE assays.id = ?").get(mpAssay.getId()) + ); if (!row && this.study) { // Try to find information about this assay by name and study id. - row = this.db.prepare("SELECT * FROM assays WHERE name = ? AND study_id = ?").get( - mpAssay.getName(), - this.study.getId() - ); + row = await this.dbManager.performQuery((db: Database) => { + return db.prepare("SELECT * FROM assays WHERE name = ? AND study_id = ?").get( + mpAssay.getName(), + this.study.getId() + ); + }) } - if (row) { mpAssay.id = row.id; - const metadataRow = this.db.prepare( - "SELECT * FROM storage_metadata WHERE assay_id = ?" - ).get(mpAssay.getId()); + const metadataRow = await this.dbManager.performQuery((db: Database) => { + return db.prepare( + "SELECT * FROM storage_metadata WHERE assay_id = ?" + ).get(mpAssay.getId()); + }) let config: SearchConfiguration; @@ -43,7 +49,7 @@ export default class AssayFileSystemMetaDataReader extends FileSystemAssayVisito false, metadataRow.configuration_id ); - const configReader = new SearchConfigFileSystemReader(this.db); + const configReader = new SearchConfigFileSystemReader(this.dbManager); configReader.visitSearchConfiguration(config); } else { config = new SearchConfiguration(); diff --git a/src/logic/filesystem/assay/AssayFileSystemMetaDataWriter.ts b/src/logic/filesystem/assay/AssayFileSystemMetaDataWriter.ts index 25db488d..a8717c70 100644 --- a/src/logic/filesystem/assay/AssayFileSystemMetaDataWriter.ts +++ b/src/logic/filesystem/assay/AssayFileSystemMetaDataWriter.ts @@ -2,45 +2,48 @@ import FileSystemAssayVisitor from "@/logic/filesystem/assay/FileSystemAssayVisi import { Database, RunResult } from "better-sqlite3"; import { Study, ProteomicsAssay, SearchConfiguration } from "unipept-web-components"; import SearchConfigFileSystemWriter from "@/logic/filesystem/configuration/SearchConfigFileSystemWriter"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; export class AssayFileSystemMetaDataWriter extends FileSystemAssayVisitor { protected readonly study: Study; - public constructor(directoryPath: string, db: Database, study: Study) { - super(directoryPath, db); + public constructor(directoryPath: string, dbManager: DatabaseManager, study: Study) { + super(directoryPath, dbManager); this.study = study } public async visitProteomicsAssay(mpAssay: ProteomicsAssay): Promise { - this.saveAssayMetaData(mpAssay); + await this.saveAssayMetaData(mpAssay); } - private saveAssayMetaData(mpAssay: ProteomicsAssay): void { + private async saveAssayMetaData(mpAssay: ProteomicsAssay): Promise { let searchConfig = mpAssay.getSearchConfiguration(); if (!searchConfig) { searchConfig = new SearchConfiguration(); mpAssay.setSearchConfiguration(searchConfig); } - const searchConfigWriter = new SearchConfigFileSystemWriter(this.db); + const searchConfigWriter = new SearchConfigFileSystemWriter(this.dbManager); searchConfigWriter.visitSearchConfiguration(searchConfig); - this.db.prepare( - "REPLACE INTO assays (id, name, study_id) VALUES (?, ?, ?)" - ).run( - mpAssay.getId(), - mpAssay.getName(), - this.study.getId() - ); + await this.dbManager.performQuery((db: Database) => { + db.prepare( + "REPLACE INTO assays (id, name, study_id) VALUES (?, ?, ?)" + ).run( + mpAssay.getId(), + mpAssay.getName(), + this.study.getId() + ); - this.db.prepare( - "REPLACE INTO storage_metadata (assay_id, configuration_id, endpoint, db_version, analysis_date) VALUES (?, ?, ?, ?, ?)" - ).run( - mpAssay.getId(), - searchConfig.id, - mpAssay.getEndpoint(), - mpAssay.getDatabaseVersion(), - mpAssay.getDate().toJSON() - ) + db.prepare( + "REPLACE INTO storage_metadata (assay_id, configuration_id, endpoint, db_version, analysis_date) VALUES (?, ?, ?, ?, ?)" + ).run( + mpAssay.getId(), + searchConfig.id, + mpAssay.getEndpoint(), + mpAssay.getDatabaseVersion(), + mpAssay.getDate().toJSON() + ) + }); } } diff --git a/src/logic/filesystem/assay/FileSystemAssayChangeListener.ts b/src/logic/filesystem/assay/FileSystemAssayChangeListener.ts index b1821d08..fe262df2 100644 --- a/src/logic/filesystem/assay/FileSystemAssayChangeListener.ts +++ b/src/logic/filesystem/assay/FileSystemAssayChangeListener.ts @@ -14,8 +14,6 @@ export default class FileSystemAssayChangeListener implements ChangeListener { - console.log("Update assay..."); - console.log("HUPLA..."); if (["date", "searchConfiguration", "endpoint", "databaseVersion"].indexOf(field) !== -1) { // Only update metadata in this case await this.serializeMetaData(object); @@ -31,7 +29,7 @@ export default class FileSystemAssayChangeListener implements ChangeListener { const mdWriter: FileSystemAssayVisitor = new AssayFileSystemMetaDataWriter( this.getAssayDirectory(), - store.getters.database, + store.getters.dbManager, this.study ); @@ -50,7 +48,7 @@ export default class FileSystemAssayChangeListener implements ChangeListener { const writer: FileSystemAssayVisitor = new AssayFileSystemMetaDataWriter( this.getAssayDirectory(), - store.getters.database, + store.getters.dbManager, this.study ); @@ -60,7 +58,7 @@ export default class FileSystemAssayChangeListener implements ChangeListener { const writer: FileSystemAssayVisitor = new AssayFileSystemDataWriter( this.getAssayDirectory(), - store.getters.database + store.getters.dbManager ); await assay.accept(writer); diff --git a/src/logic/filesystem/assay/FileSystemAssayVisitor.ts b/src/logic/filesystem/assay/FileSystemAssayVisitor.ts index f95f72fa..7ea3a44f 100644 --- a/src/logic/filesystem/assay/FileSystemAssayVisitor.ts +++ b/src/logic/filesystem/assay/FileSystemAssayVisitor.ts @@ -1,5 +1,6 @@ import { Database } from "better-sqlite3"; import { AssayVisitor, ProteomicsAssay } from "unipept-web-components"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; /** * A specific kind of visitor for assays that's specifically tailored at storing and reading information from the @@ -7,18 +8,18 @@ import { AssayVisitor, ProteomicsAssay } from "unipept-web-components"; */ export default abstract class FileSystemAssayVisitor implements AssayVisitor { protected directoryPath: string; - protected readonly db: Database; + protected readonly dbManager: DatabaseManager; /** * @param directoryPath path to the parent directory of this assay. - * @param db The database-object that keeps track of metadata for the assays. + * @param dbManager The database-object that keeps track of metadata for the assays. */ - constructor(directoryPath: string, db: Database) { + constructor(directoryPath: string, dbManager: DatabaseManager) { if (!directoryPath.endsWith("/")) { directoryPath += "/"; } this.directoryPath = directoryPath; - this.db = db; + this.dbManager = dbManager; } public abstract visitProteomicsAssay(mpAssay: ProteomicsAssay): Promise; diff --git a/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts b/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts index 97b830c1..884de468 100644 --- a/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts +++ b/src/logic/filesystem/assay/processed/ProcessedAssayManager.ts @@ -3,10 +3,7 @@ import { SearchConfiguration, PeptideTrust, PeptideData, - PeptideDataSerializer, - NetworkConfiguration, - DateUtils, - QueueManager + PeptideDataSerializer } from "unipept-web-components"; import ProcessedAssayResult from "@/logic/filesystem/assay/processed/ProcessedAssayResult"; @@ -14,13 +11,11 @@ import { Database } from "better-sqlite3"; import SearchConfigFileSystemReader from "@/logic/filesystem/configuration/SearchConfigFileSystemReader"; import { ShareableMap } from "shared-memory-datastructures"; import SearchConfigFileSystemWriter from "@/logic/filesystem/configuration/SearchConfigFileSystemWriter"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; export default class ProcessedAssayManager { - private static worker: any; - constructor( - private readonly db: Database, - private readonly dbFile: string + private readonly dbManager: DatabaseManager, ) {} /** @@ -31,7 +26,9 @@ export default class ProcessedAssayManager { */ public async readProcessingResults(assay: ProteomicsAssay): Promise { // Look up whether storage metadata with the given properties is present in the database. - const row = this.db.prepare("SELECT * FROM storage_metadata WHERE assay_id = ?").get(assay.getId()); + const row = await this.dbManager.performQuery((db: Database) => { + return db.prepare("SELECT * FROM storage_metadata WHERE assay_id = ?").get(assay.getId()); + }); if (!row) { return null; @@ -43,7 +40,7 @@ export default class ProcessedAssayManager { false, row.configuration_id ); - const searchConfigReader = new SearchConfigFileSystemReader(this.db); + const searchConfigReader = new SearchConfigFileSystemReader(this.dbManager); await serializedSearchConfig.accept(searchConfigReader); // Now check whether the search config is the same as the one that's currently assigned to this assay. @@ -56,29 +53,31 @@ export default class ProcessedAssayManager { return null; } - // Now try to read the serialized pept2data from the database - const result = await QueueManager.getLongRunningQueue().pushTask< - [ - ArrayBuffer, - ArrayBuffer, - PeptideTrust - ] | null, - [ - string, - string, - string - ] - >("readPept2Data", [ - __dirname, - this.dbFile, - assay.getId() - ]); - - if (!result) { + const dataRow = await this.dbManager.performQuery((db: Database) => { + return db.prepare("SELECT * FROM pept2data WHERE `assay_id` = ?").get(assay.getId()); + }) + + if (!dataRow) { return null; } - const [indexBuffer, dataBuffer, trust] = result; + const indexBuffer = this.bufferToSharedArrayBuffer(dataRow.index_buffer); + const dataBuffer = this.bufferToSharedArrayBuffer(dataRow.data_buffer); + + const trustRow = await this.dbManager.performQuery((db: Database) => { + return db.prepare("SELECT * FROM peptide_trust WHERE `assay_id` = ?").get(assay.getId()); + }) + + if (!trustRow) { + return null; + } + + const trust = new PeptideTrust( + JSON.parse(trustRow.missed_peptides), + trustRow.matched_peptides, + trustRow.searched_peptides + ); + const sharedMap = new ShareableMap(0, 0, new PeptideDataSerializer()); sharedMap.setBuffers( indexBuffer, @@ -101,21 +100,31 @@ export default class ProcessedAssayManager { trust: PeptideTrust ) { // Delete the metadata that's associated with this assay - this.db.prepare("DELETE FROM storage_metadata WHERE assay_id = ?").run(assay.getId()); + await this.dbManager.performQuery((db: Database) => { + db.prepare("DELETE FROM storage_metadata WHERE assay_id = ?").run(assay.getId()) + }); // First try to write the pept2data information to the database. const buffers = pept2Data.getBuffers(); - await QueueManager.getLongRunningQueue().pushTask< - void, - [string, ArrayBuffer, ArrayBuffer, PeptideTrust, string, string] - >("writePept2Data", [ - __dirname, - buffers[0], - buffers[1], - trust, - assay.getId(), - this.dbFile - ]); + await this.dbManager.performQuery((db: Database) => { + // First delete all existing rows for this assay; + db.prepare("DELETE FROM pept2data WHERE `assay_id` = ?").run(assay.getId()); + db.prepare("DELETE FROM peptide_trust WHERE `assay_id` = ?").run(assay.getId()); + + db.prepare("INSERT INTO pept2data VALUES (?, ?, ?)").run( + assay.getId(), + this.arrayBufferToBuffer(buffers[0]), + this.arrayBufferToBuffer(buffers[1]) + ); + + const insertTrust = db.prepare("INSERT INTO peptide_trust VALUES (?, ?, ?, ?)"); + insertTrust.run( + assay.getId(), + JSON.stringify(trust.missedPeptides), + trust.matchedPeptides, + trust.searchedPeptides + ); + }); // Now write the metadata to the database again. const existingConfig = assay.getSearchConfiguration(); @@ -125,15 +134,30 @@ export default class ProcessedAssayManager { existingConfig.enableMissingCleavageHandling ); - const searchConfigWriter = new SearchConfigFileSystemWriter(this.db); + const searchConfigWriter = new SearchConfigFileSystemWriter(this.dbManager); searchConfigWriter.visitSearchConfiguration(searchConfiguration); - this.db.prepare("INSERT INTO storage_metadata VALUES (?, ?, ?, ?, ?)").run( - assay.getId(), - searchConfiguration.id, - assay.getEndpoint(), - assay.getDatabaseVersion(), - assay.getDate().toJSON() - ); + await this.dbManager.performQuery((db: Database) => { + db.prepare("INSERT INTO storage_metadata VALUES (?, ?, ?, ?, ?)").run( + assay.getId(), + searchConfiguration.id, + assay.getEndpoint(), + assay.getDatabaseVersion(), + assay.getDate().toJSON() + ); + }); + } + + private arrayBufferToBuffer(buffer: ArrayBuffer): Buffer { + return Buffer.from(buffer); + } + + private bufferToSharedArrayBuffer(buf: Buffer): SharedArrayBuffer { + const ab = new SharedArrayBuffer(buf.length); + const view = new Uint8Array(ab); + for (let i = 0; i < buf.length; ++i) { + view[i] = buf[i]; + } + return ab; } } diff --git a/src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts b/src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts deleted file mode 100644 index f5fd9a64..00000000 --- a/src/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource.ts +++ /dev/null @@ -1,78 +0,0 @@ -import PeptideTrust from "unipept-web-components/src/business/processors/raw/PeptideTrust"; -import Database from "better-sqlite3"; - - -export async function readPept2Data( - [installationDir, dbFile, assayId]: [string, string, string] -): Promise<[ArrayBuffer, ArrayBuffer, PeptideTrust] | null> { - // @ts-ignore - const db = new Database(dbFile, { timeout: 15000 }, installationDir); - - const row = db.prepare("SELECT * FROM pept2data WHERE `assay_id` = ?").get(assayId); - - if (!row) { - return null; - } - - const indexBuffer = bufferToSharedArrayBuffer(row.index_buffer); - const dataBuffer = bufferToSharedArrayBuffer(row.data_buffer); - - const trustRow = db.prepare("SELECT * FROM peptide_trust WHERE `assay_id` = ?").get(assayId); - - if (!trustRow) { - return null; - } - - const peptideTrust = new PeptideTrust( - JSON.parse(trustRow.missed_peptides), - trustRow.matched_peptides, - trustRow.searched_peptides - ); - - return [indexBuffer, dataBuffer, peptideTrust] -} - -export async function writePept2Data( - [ - installationDir, - peptDataIndexBuffer, - peptDataDataBuffer, - peptideTrust, - assayId, - dbFile - ]: [string, ArrayBuffer, ArrayBuffer, PeptideTrust, string, string] -) { - //@ts-ignore - const db = new Database(dbFile, { timeout: 15000 }, installationDir); - - // First delete all existing rows for this assay; - db.prepare("DELETE FROM pept2data WHERE `assay_id` = ?").run(assayId); - db.prepare("DELETE FROM peptide_trust WHERE `assay_id` = ?").run(assayId); - - db.prepare("INSERT INTO pept2data VALUES (?, ?, ?)").run( - assayId, - arrayBufferToBuffer(peptDataIndexBuffer), - arrayBufferToBuffer(peptDataDataBuffer) - ); - - const insertTrust = db.prepare("INSERT INTO peptide_trust VALUES (?, ?, ?, ?)"); - insertTrust.run( - assayId, - JSON.stringify(peptideTrust.missedPeptides), - peptideTrust.matchedPeptides, - peptideTrust.searchedPeptides - ); -} - -function arrayBufferToBuffer(buffer: ArrayBuffer): Buffer { - return new Buffer(buffer); -} - -function bufferToSharedArrayBuffer(buf: Buffer): SharedArrayBuffer { - const ab = new SharedArrayBuffer(buf.length); - const view = new Uint8Array(ab); - for (let i = 0; i < buf.length; ++i) { - view[i] = buf[i]; - } - return ab; -} diff --git a/src/logic/filesystem/configuration/SearchConfigFileSystemDestroyer.ts b/src/logic/filesystem/configuration/SearchConfigFileSystemDestroyer.ts index cdfd9b1d..6769b521 100644 --- a/src/logic/filesystem/configuration/SearchConfigFileSystemDestroyer.ts +++ b/src/logic/filesystem/configuration/SearchConfigFileSystemDestroyer.ts @@ -1,5 +1,6 @@ import { Database } from "better-sqlite3"; import { SearchConfiguration, SearchConfigurationVisitor } from "unipept-web-components"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; /** * Remove a search configuration from a database. If no search configuration with the given id exists, no error will be @@ -9,18 +10,14 @@ import { SearchConfiguration, SearchConfigurationVisitor } from "unipept-web-com */ export default class SearchConfigFileSystemDestroyer implements SearchConfigurationVisitor { constructor( - private readonly db: Database + private readonly dbManager: DatabaseManager ) {} public async visitSearchConfiguration(config: SearchConfiguration): Promise { - // First check if a config with the given id is present in the database. if (config.id) { - const results = this.db.prepare("SELECT * FROM search_configuration WHERE id = ?").get(config.id); - - if (results) { - // It is present, now we can remove it without problems. - this.db.prepare("DELETE FROM search_configuration WHERE id = ?").run(config.id); - } + await this.dbManager.performQuery( + (db: Database) => db.prepare("DELETE FROM search_configuration WHERE id = ?").run(config.id) + ); } // An undefined id means that the configuration is no longer present in the database. diff --git a/src/logic/filesystem/configuration/SearchConfigFileSystemReader.ts b/src/logic/filesystem/configuration/SearchConfigFileSystemReader.ts index ce526274..17bdca3d 100644 --- a/src/logic/filesystem/configuration/SearchConfigFileSystemReader.ts +++ b/src/logic/filesystem/configuration/SearchConfigFileSystemReader.ts @@ -1,5 +1,6 @@ import { SearchConfigurationVisitor, SearchConfiguration } from "unipept-web-components"; import { Database } from "better-sqlite3"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; /** * Read a search configuration from a database. The search configuration's id must have been filled in for this to work. @@ -9,12 +10,15 @@ import { Database } from "better-sqlite3"; */ export default class SearchConfigFileSystemReader implements SearchConfigurationVisitor { constructor( - private readonly db: Database + private readonly dbManager: DatabaseManager ) {} public async visitSearchConfiguration(config: SearchConfiguration): Promise { if (config.id) { - const result = this.db.prepare("SELECT * FROM search_configuration WHERE id = ?").get(config.id); + const result = await this.dbManager.performQuery((db: Database) => { + db.prepare("SELECT * FROM search_configuration WHERE id = ?").get(config.id); + }); + if (result) { config.equateIl = (result.equate_il === 1); config.filterDuplicates = (result.filter_duplicates === 1); diff --git a/src/logic/filesystem/configuration/SearchConfigFileSystemWriter.ts b/src/logic/filesystem/configuration/SearchConfigFileSystemWriter.ts index ba976e7f..a6bb0b3f 100644 --- a/src/logic/filesystem/configuration/SearchConfigFileSystemWriter.ts +++ b/src/logic/filesystem/configuration/SearchConfigFileSystemWriter.ts @@ -1,5 +1,6 @@ import { SearchConfigurationVisitor, SearchConfiguration } from "unipept-web-components"; import { Database, RunResult } from "better-sqlite3"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; /** * Writes a search configuration to a database. If the configuration's id is undefined, a new config will be created, @@ -9,7 +10,7 @@ import { Database, RunResult } from "better-sqlite3"; */ export default class SearchConfigFileSystemWriter implements SearchConfigurationVisitor { constructor( - private readonly db: Database + private readonly dbManager: DatabaseManager ) {} public async visitSearchConfiguration(config: SearchConfiguration): Promise { @@ -17,29 +18,36 @@ export default class SearchConfigFileSystemWriter implements SearchConfiguration // Check if the search configuration already exists in the database. if (config.id) { - const result = this.db.prepare("SELECT * FROM search_configuration WHERE id = ?").get(config.id); + const result = await this.dbManager.performQuery((db: Database) => { + return db.prepare("SELECT * FROM search_configuration WHERE id = ?").get(config.id); + }); + if (result) { insertNew = false; } } if (insertNew) { - const info: RunResult = this.db.prepare( - ` + await this.dbManager.performQuery((db: Database) => { + const info: RunResult = db.prepare( + ` INSERT INTO search_configuration (equate_il, filter_duplicates, missing_cleavage_handling) VALUES (?, ?, ?) ` - ).run( - config.equateIl ? 1 : 0, - config.filterDuplicates ? 1 : 0, - config.enableMissingCleavageHandling ? 1: 0 - ); + ).run( + config.equateIl ? 1 : 0, + config.filterDuplicates ? 1 : 0, + config.enableMissingCleavageHandling ? 1: 0 + ); - config.id = info.lastInsertRowid.toString(); + config.id = info.lastInsertRowid.toString(); + }); } else { - this.db.prepare( - "UPDATE search_configuration SET equate_il = ?, filter_duplicates = ?, missing_cleavage_handling = ? WHERE `id` = ?" - ).run(config.equateIl ? 1 : 0, config.filterDuplicates ? 1 : 0, config.enableMissingCleavageHandling ? 1 : 0, config.id); + await this.dbManager.performQuery((db: Database) => { + db.prepare( + "UPDATE search_configuration SET equate_il = ?, filter_duplicates = ?, missing_cleavage_handling = ? WHERE `id` = ?" + ).run(config.equateIl ? 1 : 0, config.filterDuplicates ? 1 : 0, config.enableMissingCleavageHandling ? 1 : 0, config.id); + }); } } } diff --git a/src/logic/filesystem/database/DatabaseManager.ts b/src/logic/filesystem/database/DatabaseManager.ts new file mode 100644 index 00000000..c861fdc0 --- /dev/null +++ b/src/logic/filesystem/database/DatabaseManager.ts @@ -0,0 +1,32 @@ +import Database, { Database as DbType } from "better-sqlite3"; +import async, { AsyncQueue } from "async"; + +export default class DatabaseManager { + // Reading and writing large assays to and from the database can easily take longer than 5 seconds, causing + // a "SQLBusyException" to b§e thrown. By increasing the timeout to a value, larger than the time it should take + // to execute these transactions, these errors can be avoided. + public static readonly DB_TIMEOUT: number = 15000; + + private readonly db: DbType; + private readonly queue: AsyncQueue; + + constructor( + private readonly dbLocation: string + ) { + this.db = new Database(this.dbLocation, { + timeout: DatabaseManager.DB_TIMEOUT + }); + + this.queue = async.queue((task, callback) => { + callback(task.query(this.db)); + }, 1); + } + + public async performQuery(query: (db: DbType) => ResultType): Promise { + return new Promise((resolve) => { + this.queue.push({ + query + }, resolve); + }); + } +} diff --git a/src/logic/filesystem/project/FileSystemWatcher.ts b/src/logic/filesystem/project/FileSystemWatcher.ts index 55525b0a..78867031 100644 --- a/src/logic/filesystem/project/FileSystemWatcher.ts +++ b/src/logic/filesystem/project/FileSystemWatcher.ts @@ -89,7 +89,7 @@ export default class FileSystemWatcher { // Read metadata from disk if it exists. const assayMetaReader = new AssayFileSystemMetaDataReader( store.getters.projectLocation + studyName, - store.getters.database, + store.getters.dbManager, study ); await assay.accept(assayMetaReader); @@ -97,7 +97,7 @@ export default class FileSystemWatcher { // Read peptides from disk for this assay const assayReader: FileSystemAssayVisitor = new AssayFileSystemDataReader( store.getters.projectLocation + studyName, - store.getters.database + store.getters.dbManager ); await assay.accept(assayReader); @@ -105,7 +105,7 @@ export default class FileSystemWatcher { // Write metadata for this assay to disk const assayWriter = new AssayFileSystemMetaDataWriter( store.getters.projectPath + studyName, - store.getters.database, + store.getters.dbManager, study ); @@ -137,11 +137,14 @@ export default class FileSystemWatcher { const study: Study = new Study(uuidv4()); study.setName(studyName); - const studyWriter: FileSystemStudyVisitor = new StudyFileSystemMetaDataWriter(directoryPath, store.getters.database); + const studyWriter: FileSystemStudyVisitor = new StudyFileSystemMetaDataWriter( + directoryPath, + store.getters.dbManager + ); await study.accept(studyWriter); // This reader directly reads all assays associated with this study from disk. - const studyReader = new StudyFileSystemDataReader(directoryPath, store.getters.database); + const studyReader = new StudyFileSystemDataReader(directoryPath, store.getters.dbManager); await study.accept(studyReader); await store.dispatch("addStudy", study); @@ -180,7 +183,7 @@ export default class FileSystemWatcher { const dataReader: FileSystemAssayVisitor = new AssayFileSystemDataReader( path.dirname(filePath), - store.getters.database + store.getters.dbManager ); // This assay's change listener should be active at this point and should reprocess automatically. @@ -209,8 +212,7 @@ export default class FileSystemWatcher { await study.removeAssay(assay); const assayDestroyer = new AssayFileSystemDestroyer( store.getters.projectLocation + studyName, - store.getters.database, - store.getters.databaseFile + store.getters.dbManager ); await assay.accept(assayDestroyer); @@ -236,8 +238,7 @@ export default class FileSystemWatcher { const assayDestroyer = new AssayFileSystemDestroyer( store.getters.projectLocation + studyName, - store.getters.database, - store.getters.databaseFile + store.getters.dbManager ); for (const assay of study.getAssays()) { @@ -246,7 +247,7 @@ export default class FileSystemWatcher { const studyDestroyer = new StudyFileSystemRemover( store.getters.projectLocation + studyName, - store.getters.database + store.getters.dbManager ); await study.accept(studyDestroyer); diff --git a/src/logic/filesystem/project/ProjectManager.ts b/src/logic/filesystem/project/ProjectManager.ts index c389caaf..3e791db3 100644 --- a/src/logic/filesystem/project/ProjectManager.ts +++ b/src/logic/filesystem/project/ProjectManager.ts @@ -7,11 +7,12 @@ import schema_v1 from "raw-loader!@/db/schemas/schema_v1.sql"; import StudyFileSystemDataReader from "@/logic/filesystem/study/StudyFileSystemDataReader"; import RecentProjectsManager from "@/logic/filesystem/project/RecentProjectsManager"; import { Study, IOException } from "unipept-web-components"; -import Database, { Database as DatabaseType } from "better-sqlite3"; +import { Database as DatabaseType } from "better-sqlite3"; import { v4 as uuidv4 } from "uuid"; import StudyFileSystemMetaDataWriter from "@/logic/filesystem/study/StudyFileSystemMetaDataWriter"; import FileSystemStudyChangeListener from "@/logic/filesystem/study/FileSystemStudyChangeListener"; import FileSystemAssayChangeListener from "@/logic/filesystem/assay/FileSystemAssayChangeListener"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; export default class ProjectManager { @@ -35,10 +36,7 @@ export default class ProjectManager { throw new InvalidProjectException("Project metadata file was not found!"); } - const db = new Database(projectLocation + ProjectManager.DB_FILE_NAME, { - timeout: ProjectManager.DB_TIMEOUT, - verbose: console.warn - }); + const dbManager = new DatabaseManager(projectLocation + ProjectManager.DB_FILE_NAME); // Check all subdirectories of the given project and try to load the studies. const subDirectories: string[] = fs.readdirSync(projectLocation, { withFileTypes: true }) @@ -48,12 +46,12 @@ export default class ProjectManager { const studies = []; for (const directory of subDirectories) { - studies.push(await this.loadStudy(`${projectLocation}${directory}`, db)); + studies.push(await this.loadStudy(`${projectLocation}${directory}`, dbManager)); } await this.addToRecentProjects(projectLocation); - await store.dispatch("initializeProject", [projectLocation, db, studies]); + await store.dispatch("initializeProject", [projectLocation, dbManager, studies]); for (const study of studies) { for (const assay of study.getAssays()) { @@ -72,18 +70,17 @@ export default class ProjectManager { projectLocation += "/"; } - const db = new Database(projectLocation + ProjectManager.DB_FILE_NAME, { - timeout: ProjectManager.DB_TIMEOUT, - verbose: console.warn - }); - db.exec(schema_v1); + const dbManager = new DatabaseManager(projectLocation + ProjectManager.DB_FILE_NAME); + await dbManager.performQuery((db: DatabaseType) => { + db.exec(schema_v1); + }) await this.addToRecentProjects(projectLocation); - await store.dispatch("initializeProject", [projectLocation, db, []]); + await store.dispatch("initializeProject", [projectLocation, dbManager, []]); } - private async loadStudy(directory: string, db: DatabaseType): Promise { + private async loadStudy(directory: string, dbManager: DatabaseManager): Promise { if (!directory.endsWith("/")) { directory += "/"; } @@ -92,7 +89,9 @@ export default class ProjectManager { let study: Study; // Check if the given study name is present in the database. If not, add the study with a new ID. - const row = db.prepare("SELECT * FROM studies WHERE `name`=?").get(studyName); + const row = await dbManager.performQuery((db: DatabaseType) => { + return db.prepare("SELECT * FROM studies WHERE `name`=?").get(studyName); + }) if (row) { study = new Study(row.id); @@ -102,11 +101,11 @@ export default class ProjectManager { study.setName(studyName); - const studyWriter = new StudyFileSystemMetaDataWriter(directory, db); + const studyWriter = new StudyFileSystemMetaDataWriter(directory, dbManager); await study.accept(studyWriter); // Read all assays from this study - const studyReader = new StudyFileSystemDataReader(directory, db); + const studyReader = new StudyFileSystemDataReader(directory, dbManager); await study.accept(studyReader); return study; diff --git a/src/logic/filesystem/study/FileSystemStudyChangeListener.ts b/src/logic/filesystem/study/FileSystemStudyChangeListener.ts index 6d8547e7..a1af9664 100644 --- a/src/logic/filesystem/study/FileSystemStudyChangeListener.ts +++ b/src/logic/filesystem/study/FileSystemStudyChangeListener.ts @@ -32,7 +32,7 @@ export default class FileSystemStudyChangeListener implements ChangeListener { const assayRemover: FileSystemAssayVisitor = new AssayFileSystemDestroyer( `${store.getters.projectLocation}${study.getName()}`, - store.getters.database, - store.getters.databaseFile + store.getters.dbManager ); await assay.accept(assayRemover); @@ -52,10 +51,10 @@ export default class FileSystemStudyChangeListener implements ChangeListener((db: Database) => { + return db.prepare( + "SELECT * FROM assays WHERE `name`=? and `study_id`=?" + ).get(assayName, study.getId()); + }); if (row) { // Assay exists. Get it's ID and create a new object. assay = new ProteomicsAssay(row.id); assay.setName(assayName); - const assayVisitor = new AssayFileSystemMetaDataReader(this.studyPath, this.db); + const assayVisitor = new AssayFileSystemMetaDataReader(this.studyPath, this.dbManager); await assay.accept(assayVisitor); } else { // If assay not present in metadata, create a new UUID and write it to metadata. @@ -46,7 +49,7 @@ export default class StudyFileSystemDataReader extends FileSystemStudyVisitor { const assayVisitor: AssayVisitor = new AssayFileSystemMetaDataWriter( this.studyPath, - this.db, + this.dbManager, study ); await assay.accept(assayVisitor); @@ -54,7 +57,7 @@ export default class StudyFileSystemDataReader extends FileSystemStudyVisitor { // Also read in any data related to this assay. try { - const dataReader: AssayVisitor = new AssayFileSystemDataReader(this.studyPath, this.db); + const dataReader: AssayVisitor = new AssayFileSystemDataReader(this.studyPath, this.dbManager); await assay.accept(dataReader); study.addAssay(assay); diff --git a/src/logic/filesystem/study/StudyFileSystemMetaDataReader.ts b/src/logic/filesystem/study/StudyFileSystemMetaDataReader.ts index 6ad43115..4b00fb54 100644 --- a/src/logic/filesystem/study/StudyFileSystemMetaDataReader.ts +++ b/src/logic/filesystem/study/StudyFileSystemMetaDataReader.ts @@ -1,11 +1,14 @@ import FileSystemStudyVisitor from "@/logic/filesystem/study/FileSystemStudyVisitor"; import { Study, IOException } from "unipept-web-components"; +import { Database } from "better-sqlite3"; export default class StudyFileSystemMetaDataReader extends FileSystemStudyVisitor { public async visitStudy(study: Study): Promise { try { if (study.getId()) { - const row = this.db.prepare("SELECT * FROM studies WHERE id = ?").get(study.getId()); + const row = await this.dbManager.performQuery((db: Database) => { + return db.prepare("SELECT * FROM studies WHERE id = ?").get(study.getId()); + }); if (row) { study.setName(row.name); diff --git a/src/logic/filesystem/study/StudyFileSystemMetaDataWriter.ts b/src/logic/filesystem/study/StudyFileSystemMetaDataWriter.ts index b9f612fa..60b8a36c 100644 --- a/src/logic/filesystem/study/StudyFileSystemMetaDataWriter.ts +++ b/src/logic/filesystem/study/StudyFileSystemMetaDataWriter.ts @@ -1,6 +1,7 @@ import FileSystemStudyVisitor from "./FileSystemStudyVisitor"; import mkdirp from "mkdirp"; import { Study, IOException } from "unipept-web-components"; +import { Database } from "better-sqlite3"; /** @@ -12,16 +13,9 @@ export default class StudyFileSystemMetaDataWriter extends FileSystemStudyVisito // Make study directory if it does not exist yet... await mkdirp(`${this.studyPath}`); - if (this.db.prepare("SELECT * FROM studies WHERE `id`=?").get(study.getId())) { - this.db.prepare("UPDATE studies SET `name`=? WHERE `id`=?").run(study.getName(), study.getId()); - } else { - this.db.prepare( - "INSERT INTO studies (id, name) VALUES (?, ?)" - ).run( - study.getId(), - study.getName() - ); - } + await this.dbManager.performQuery((db: Database) => { + db.prepare("REPLACE INTO studies (id, name) VALUES (?, ?)").run(study.getId(), study.getName()) + }); } catch (err) { throw new IOException(err); } diff --git a/src/logic/system/DesktopWorker.worker.ts b/src/logic/system/DesktopWorker.worker.ts deleted file mode 100644 index 702bfec0..00000000 --- a/src/logic/system/DesktopWorker.worker.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { createMessageEventListener, workerFunctionMap } from "unipept-web-components"; -import { - compute as computeCachedEcResponse -} from "@/logic/communication/functional/CachedEcResponseCommunicator.workerSource"; -import { - compute as computeCachedGoResponse -} from "@/logic/communication/functional/CachedGoResponseCommunicator.workerSource"; -import { - compute as computeCachedInterproResponse -} from "@/logic/communication/functional/CachedInterproResponseCommunicator.workerSource"; -import { - compute as computeCachedNcbiResponse -} from "@/logic/communication/taxonomic/ncbi/CachedNcbiResponseCommunicator.workerSource"; -import { - compute as readAssay -} from "@/logic/filesystem/assay/AssayFileSystemDataReader.workerSource"; -import { - compute as destroyAssay -} from "@/logic/filesystem/assay/AssayFileSystemDestroyer.workerSource"; -import { - readPept2Data, - writePept2Data -} from "@/logic/filesystem/assay/processed/ProcessedAssayManager.workerSource"; - -workerFunctionMap.set("computeCachedEcResponses", computeCachedEcResponse); -workerFunctionMap.set("computeCachedGoResponses", computeCachedGoResponse); -workerFunctionMap.set("computeCachedInterproResponses", computeCachedInterproResponse); -workerFunctionMap.set("computeCachedNcbiResponses", computeCachedNcbiResponse); -workerFunctionMap.set("readAssay", readAssay); -workerFunctionMap.set("destroyAssay", destroyAssay); -workerFunctionMap.set("readPept2Data", readPept2Data); -workerFunctionMap.set("writePept2Data", writePept2Data); - -const ctx: Worker = self as any; - -// Respond to message from parent thread -ctx.addEventListener("message", createMessageEventListener(ctx)); diff --git a/src/main.ts b/src/main.ts index 57c3fd28..c1c2162e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -75,7 +75,7 @@ const assayStore = createAssayStore(( assay: ProteomicsAssay, progressListener: ProgressListener ) => { - return new DesktopAssayProcessor(store.getters["database"], store.getters["databaseFile"], assay, progressListener); + return new DesktopAssayProcessor(store.getters.dbManager, assay, progressListener); }); diff --git a/src/state/PeptideSummaryStore.ts b/src/state/PeptideSummaryStore.ts index b167d6d5..b9f93cde 100644 --- a/src/state/PeptideSummaryStore.ts +++ b/src/state/PeptideSummaryStore.ts @@ -1,4 +1,3 @@ -import { spawn, Worker } from "threads/dist"; import { ActionContext, ActionTree, GetterTree, MutationTree } from "vuex"; import ProteomicsAssay from "unipept-web-components/src/business/entities/assay/ProteomicsAssay"; import { DataOptions } from "vuetify"; @@ -8,6 +7,7 @@ import NcbiTaxon from "unipept-web-components/src/business/ontology/taxonomic/nc import { NcbiId } from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiTaxon"; import { CountTable } from "unipept-web-components/src/business/counts/CountTable"; import { Peptide } from "unipept-web-components/src/business/ontology/raw/Peptide"; +import Worker from "worker-loader?inline=fallback!./PeptideSummaryTable.worker" export interface SummaryData { assay: ProteomicsAssay, @@ -18,7 +18,7 @@ export interface PeptideSummaryState { summaryData: SummaryData[] } -let inProgress: Promise; +let inProgress: Promise; let summaryWorker: any; const summaryState: PeptideSummaryState = { @@ -27,7 +27,31 @@ const summaryState: PeptideSummaryState = { const summaryGetters: GetterTree = { getSummaryItems(state: PeptideSummaryState): (assay: ProteomicsAssay, options: DataOptions) => Promise { - return async(assay: ProteomicsAssay, options: DataOptions) => summaryWorker.getItems(assay.getId(), options); + return async(assay: ProteomicsAssay, options: DataOptions) => { + while (inProgress) { + await inProgress; + } + + inProgress = new Promise(async(resolve, reject) => { + const eventListener = (message: MessageEvent) => { + summaryWorker.removeEventListener("message", eventListener); + resolve(message.data.result); + } + + summaryWorker.addEventListener("message", eventListener) + + summaryWorker.postMessage( + { + type: "getItems", + args: [assay.getId(), options] + } + ); + }); + + const result = await inProgress; + inProgress = undefined; + return result; + } }, getProgress(state: PeptideSummaryState): (assay: ProteomicsAssay) => number { @@ -75,7 +99,7 @@ const summaryActions: ActionTree = { inProgress = new Promise(async(resolve, reject) => { if (!summaryWorker) { - summaryWorker = await spawn(new Worker("./PeptideSummaryTable.worker.ts")); + summaryWorker = new Worker(); } const assayData = store.rootGetters["assayData"](assay); @@ -92,18 +116,24 @@ const summaryActions: ActionTree = { const responseMap = pept2DataCommunicator.getPeptideResponseMap(assay.getSearchConfiguration()); const [indexBuffer, dataBuffer] = responseMap.getBuffers(); - const obs = summaryWorker.computeItems( - assay.getId(), indexBuffer, dataBuffer, countTable.toMap(), ontology - ); + await new Promise((resolve) => { + const eventListener = (message: MessageEvent) => { + summaryWorker.removeEventListener("message", eventListener); + resolve(); + } + + summaryWorker.addEventListener("message", eventListener) - await new Promise((resolve, reject) => { - obs.subscribe( - (val: number) => store.commit("SET_PROGRESS", [assay, val]), - (err: Error) => reject(err), - () => resolve(), + summaryWorker.postMessage( + { + type: "computeItems", + args: [assay.getId(), indexBuffer, dataBuffer, countTable.toMap(), ontology] + } ); }); + store.commit("SET_PROGRESS", [assay, 1]); + resolve(); }); diff --git a/src/state/PeptideSummaryTable.worker.ts b/src/state/PeptideSummaryTable.worker.ts index 899959b1..050e0b75 100644 --- a/src/state/PeptideSummaryTable.worker.ts +++ b/src/state/PeptideSummaryTable.worker.ts @@ -1,12 +1,10 @@ import { ShareableMap } from "shared-memory-datastructures"; -import { expose } from "threads"; import NcbiTaxon from "unipept-web-components/src/business/ontology/taxonomic/ncbi/NcbiTaxon"; import { Peptide } from "unipept-web-components/src/business/ontology/raw/Peptide"; import { Ontology } from "unipept-web-components/src/business/ontology/Ontology"; import PeptideData from "unipept-web-components/src/business/communication/peptides/PeptideData"; import PeptideDataSerializer from "unipept-web-components/src/business/communication/peptides/PeptideDataSerializer"; import { DataOptions } from "vuetify"; -import { Observable } from "observable-fns"; export type ItemType = { peptide: string, @@ -15,71 +13,68 @@ export type ItemType = { rank: string }; +const ctx: Worker = self as any; + +ctx.addEventListener("message", (message: MessageEvent) => { + if (message.data.type === "computeItems") { + computeItems(message.data.args); + ctx.postMessage({ + type: "result" + }); + } else if (message.data.type === "getItems") { + ctx.postMessage({ + type: "result", + result: getItems(message.data.args) + }); + } +}); + // Maps an assay's id onto a list of all peptide summary items for this assay. const itemsPerAssay: Map = new Map(); -expose({ getItems, computeItems }); - function computeItems( - assayId: string, - indexBuffer: SharedArrayBuffer, - dataBuffer: SharedArrayBuffer, - countTable: Map, - lcaOntology: Ontology -): Observable { - return new Observable((obs) => { - const output = []; - - obs.next(0); - - const pept2DataMap: ShareableMap = new ShareableMap( - 0, - 0, - new PeptideDataSerializer() - ); - pept2DataMap.setBuffers(indexBuffer, dataBuffer); - - const totalPeptides: number = countTable.size; - let processedPeptides: number = 0; - - const start = new Date().getTime(); - - for (const peptide of countTable.keys()) { - const response: PeptideData = pept2DataMap.get(peptide); - let lcaName: string = "N/A"; - let rank: string = "N/A"; - - if (response) { - // @ts-ignore - const lcaDefinition = lcaOntology.definitions.get(response.lca); - lcaName = lcaDefinition ? lcaDefinition.name : lcaName; - rank = lcaDefinition ? lcaDefinition.rank : rank; - } - - output.push({ - peptide: peptide, - count: countTable.get(peptide), - lca: lcaName, - rank: rank - }); - - processedPeptides++; - - if (processedPeptides % 5000 === 0) { - obs.next(processedPeptides / totalPeptides); - } + [ + assayId, + indexBuffer, + dataBuffer, + countTable, + lcaOntology + ]: [string, ArrayBuffer, ArrayBuffer, Map, Ontology] +): void { + const output = []; + + const pept2DataMap: ShareableMap = new ShareableMap( + 0, + 0, + new PeptideDataSerializer() + ); + pept2DataMap.setBuffers(indexBuffer, dataBuffer); + + for (const peptide of countTable.keys()) { + const response: PeptideData = pept2DataMap.get(peptide); + let lcaName: string = "N/A"; + let rank: string = "N/A"; + + if (response) { + // @ts-ignore + const lcaDefinition = lcaOntology.definitions.get(response.lca); + lcaName = lcaDefinition ? lcaDefinition.name : lcaName; + rank = lcaDefinition ? lcaDefinition.rank : rank; } - const end = new Date().getTime(); - console.log("Peptide summary took: " + (end - start) / 1000 + "s for --> " + assayId); + output.push({ + peptide: peptide, + count: countTable.get(peptide), + lca: lcaName, + rank: rank + }); + } + - itemsPerAssay.set(assayId, output); - obs.next(1); - obs.complete(); - }); + itemsPerAssay.set(assayId, output); } -function getItems(assayId: string, options: DataOptions): ItemType[] { +function getItems([assayId, options]: [string, DataOptions]): ItemType[] { const itemsForAssay = itemsPerAssay.get(assayId); if (!itemsForAssay) { diff --git a/src/state/ProjectStore.ts b/src/state/ProjectStore.ts index 4091c96a..900dfccd 100644 --- a/src/state/ProjectStore.ts +++ b/src/state/ProjectStore.ts @@ -1,13 +1,13 @@ -import { Database } from "better-sqlite3"; import { Study } from "unipept-web-components"; import { ActionContext, ActionTree, GetterTree, MutationTree } from "vuex"; import FileSystemWatcher from "@/logic/filesystem/project/FileSystemWatcher"; import path from "path"; +import DatabaseManager from "@/logic/filesystem/database/DatabaseManager"; export interface ProjectState { projectName: string, projectLocation: string, - database: Database, + dbManager: DatabaseManager, studies: Study[], fileSystemWatcher: FileSystemWatcher } @@ -15,7 +15,7 @@ export interface ProjectState { const projectState: ProjectState = { projectName: "", projectLocation: "", - database: undefined, + dbManager: undefined, studies: [], fileSystemWatcher: undefined } @@ -33,12 +33,8 @@ const projectGetters: GetterTree = { return state.projectLocation; }, - database(state: ProjectState): Database { - return state.database; - }, - - databaseFile(state: ProjectState): string { - return state.projectLocation + "metadata.sqlite"; + dbManager(state: ProjectState): DatabaseManager { + return state.dbManager; } }; @@ -70,10 +66,10 @@ const projectMutations: MutationTree = { state.studies.push(...studies); }, - SET_PROJECT(state: ProjectState, [name, location, database]: [string, string, Database]) { + SET_PROJECT(state: ProjectState, [name, location, dbManager]: [string, string, DatabaseManager]) { state.projectName = name; state.projectLocation = location; - state.database = database; + state.dbManager = dbManager; } }; @@ -82,11 +78,11 @@ const projectActions: ActionTree = { store: ActionContext, [ projectDirectory, - database, + dbManager, studies ]: [ string, - Database, + DatabaseManager, Study[] ] ) { @@ -102,7 +98,7 @@ const projectActions: ActionTree = { const name = path.basename(projectDirectory); - store.commit("SET_PROJECT", [name, projectDirectory, database]); + store.commit("SET_PROJECT", [name, projectDirectory, dbManager]); for (const study of studies) { for (const assay of study.getAssays()) {