diff --git a/package-lock.json b/package-lock.json index a041147e..7959efac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,19 @@ { "name": "unipept-web-components", - "version": "2.0.2", + "version": "2.0.6", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "unipept-web-components", - "version": "2.0.2", + "version": "2.0.6", "dependencies": { + "@types/crypto-js": "^4.1.1", "async": "^3.2.4", "canvg": "^4.0.1", "core-js": "^3.8.3", + "crypto-js": "^4.1.1", + "dexie": "^3.2.4", "html-to-image": "^1.10.8", "jquery": "^3.6.1", "less": "^4.1.3", @@ -2092,6 +2095,11 @@ "@types/node": "*" } }, + "node_modules/@types/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA==" + }, "node_modules/@types/d3": { "version": "6.7.5", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-6.7.5.tgz", @@ -4678,6 +4686,11 @@ "semver": "bin/semver" } }, + "node_modules/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + }, "node_modules/css-declaration-sorter": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-6.3.1.tgz", @@ -5484,6 +5497,14 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "node_modules/dexie": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.4.tgz", + "integrity": "sha512-VKoTQRSv7+RnffpOJ3Dh6ozknBqzWw/F3iqMdsZg958R0AS8AnY9x9d1lbwENr0gzeGJHXKcGhAMRaqys6SxqA==", + "engines": { + "node": ">=6.0" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -13861,6 +13882,11 @@ "@types/node": "*" } }, + "@types/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-BG7fQKZ689HIoc5h+6D2Dgq1fABRa0RbBWKBd9SP/MVRVXROflpm5fhwyATX5duFmbStzyzyycPB8qUYKDH3NA==" + }, "@types/d3": { "version": "6.7.5", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-6.7.5.tgz", @@ -15870,6 +15896,11 @@ } } }, + "crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + }, "css-declaration-sorter": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-6.3.1.tgz", @@ -16500,6 +16531,11 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "dexie": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.4.tgz", + "integrity": "sha512-VKoTQRSv7+RnffpOJ3Dh6ozknBqzWw/F3iqMdsZg958R0AS8AnY9x9d1lbwENr0gzeGJHXKcGhAMRaqys6SxqA==" + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", diff --git a/package.json b/package.json index e004aa96..f31ea1e8 100644 --- a/package.json +++ b/package.json @@ -17,9 +17,12 @@ "dist/*" ], "dependencies": { + "@types/crypto-js": "^4.1.1", "async": "^3.2.4", "canvg": "^4.0.1", "core-js": "^3.8.3", + "crypto-js": "^4.1.1", + "dexie": "^3.2.4", "html-to-image": "^1.10.8", "jquery": "^3.6.1", "less": "^4.1.3", diff --git a/src/logic/communication/NetworkCacheManager.ts b/src/logic/communication/NetworkCacheManager.ts new file mode 100644 index 00000000..ee9209f8 --- /dev/null +++ b/src/logic/communication/NetworkCacheManager.ts @@ -0,0 +1,152 @@ +import NetworkUtils from "./NetworkUtils"; +import sha256 from "crypto-js/sha256"; +import Dexie from "dexie"; + +/** + * This is a specific type of request cache that can use an IndexedDB-backed request / response cache in order to speed + * reanalysing assays that have, for example, failed before. We take into account the maximum amount of storage + * space that can be used by the cache and make sure that this does not overflow. + * + * @author Pieter Verschaffelt + */ +export default class RequestCacheNetworkManager { + private indexedDb: CacheIndexedDatabase | undefined; + private uniprotVersion: string = ""; + // Epoch time at which the UniProt database version was last checked + private uniprotVersionLastChecked: number | undefined; + + // Max amount of entries in the request cache. + private static readonly MAX_REQUEST_CACHE_SIZE = 5000; + // The current UniProt-version must be revalidated every 30 seconds + private static readonly UNIPROT_VERSION_INVALIDATE_MS = 30 * 1000; + + constructor( + private readonly baseUrl: string, + private readonly cacheKey: string = "" + ) { + this.setupDb(); + } + + public async postJSON(url: string, data: any): Promise { + try { + const dbVersion: string = await this.getUniprotDBVersion(); + + const dataHash: string = sha256( + this.baseUrl + url + JSON.stringify(data) + dbVersion + this.cacheKey + ).toString(); + + const dbResult = await this.readRequestFromDb(dataHash); + if (dbResult) { + return JSON.parse(dbResult); + } + + const response = await NetworkUtils.postJSON(url, data); + await this.writeRequestToDb(dataHash, JSON.stringify(response)); + + return response; + } catch (err) { + console.warn("Error while using HTTP request / response cache: " + err); + return await NetworkUtils.postJSON(url, data); + } + } + + public async getJSON(url: string): Promise { + try { + const dbVersion: string = await this.getUniprotDBVersion(); + + const dataHash: string = sha256(this.baseUrl + url + dbVersion).toString(); + + const dbResult = await this.readRequestFromDb(dataHash); + if (dbResult) { + return JSON.parse(dbResult); + } + + const response = await NetworkUtils.getJSON(url); + await this.writeRequestToDb(dataHash, JSON.stringify(response)); + + return response; + } catch (err) { + console.warn("Error while using HTTP request / response cache: " + err); + return await NetworkUtils.getJSON(url); + } + } + + private setupDb(): void { + try { + this.indexedDb = new CacheIndexedDatabase(); + } catch (err) { + console.warn("IndexedDB storage not available. Feature has been disabled."); + } + } + + private async writeRequestToDb(key: string, response: any): Promise { + if (!this.indexedDb) { + return; + } + + await this.indexedDb.cache.put({ + hash: key, + response, + epoch: new Date().getTime() + }); + + // We need to check if the database does not contain too many entries at this point. If it grows too large, we + // will remove the oldest 100 entries in the cache to make room for more recent items. We need to check if + // either the limit set by this application is exceeded or if the storage limits provided by the browser are + // exceeded. + const quota = await this.getEstimatedQuota(); + if ( + await this.indexedDb.cache.count() > RequestCacheNetworkManager.MAX_REQUEST_CACHE_SIZE || + (quota && quota.usage && quota.quota && quota.usage * 1.5 > quota.quota) + ) { + await this.indexedDb.cache.orderBy("epoch").limit(100).delete(); + } + } + + private async readRequestFromDb(key: string): Promise { + const result = await this.indexedDb?.cache.get(key); + if (result) { + return result.response; + } else { + return undefined; + } + } + + private async getEstimatedQuota(): Promise { + return await navigator.storage && navigator.storage.estimate ? navigator.storage.estimate() : undefined; + } + + private async getUniprotDBVersion(): Promise { + const currentEpoch = new Date().getTime(); + + if ( + !this.uniprotVersion || !this.uniprotVersionLastChecked || + currentEpoch - this.uniprotVersionLastChecked > RequestCacheNetworkManager.UNIPROT_VERSION_INVALIDATE_MS + ) { + this.uniprotVersion = JSON.parse( + await NetworkUtils.get(this.baseUrl + "/private_api/metadata") + ).db_version; + + this.uniprotVersionLastChecked = currentEpoch; + } + + return this.uniprotVersion; + } +} + +class CacheIndexedDatabase extends Dexie { + cache!: Dexie.Table; + + constructor() { + super("NetworkStore"); + this.version(1).stores({ + cache: "&hash,epoch" + }); + } +} + +interface IndexedCacheRecord { + hash: string, + response: string, + epoch: number +} diff --git a/src/logic/communication/peptide/Pept2DataCommunicator.ts b/src/logic/communication/peptide/Pept2DataCommunicator.ts index 546bac05..3fd67493 100644 --- a/src/logic/communication/peptide/Pept2DataCommunicator.ts +++ b/src/logic/communication/peptide/Pept2DataCommunicator.ts @@ -9,6 +9,7 @@ import { ShareableMap } from "shared-memory-datastructures"; import NetworkUtils from "../../util/NetworkUtils"; import PeptideData from "./PeptideData"; import PeptideDataSerializer from "./PeptideDataSerializer"; +import NetworkCacheManager from "../NetworkCacheManager"; export default class Pept2DataCommunicator { // Should the analysis continue? If this flag is set to true, the analysis will be cancelled as soon as @@ -19,7 +20,8 @@ export default class Pept2DataCommunicator { private readonly apiBaseUrl: string = "http://api.unipept.ugent.be", private readonly peptdataBatchSize: number = 100, private readonly missedCleavageBatchSize: number = 25, - private readonly parallelRequests: number = 5 + private readonly parallelRequests: number = 5, + public readonly cacheKey: string = "" ) {} public async process( @@ -38,6 +40,8 @@ export default class Pept2DataCommunicator { const batchSize = enableMissingCleavageHandling ? this.missedCleavageBatchSize : this.peptdataBatchSize; + const networkManager = new NetworkCacheManager(this.apiBaseUrl, this.cacheKey); + const requests = []; for (let i = 0; i < amountOfPeptides + batchSize; i += batchSize) { requests.push(async() => { @@ -52,7 +56,7 @@ export default class Pept2DataCommunicator { }); try { - const response = await NetworkUtils.postJson(this.apiBaseUrl + "/mpa/pept2data", requestData); + const response = await networkManager.postJSON(this.apiBaseUrl + "/mpa/pept2data", requestData); for (const peptide of response.peptides) { result.set(peptide.sequence, PeptideData.createFromPeptideDataResponse(peptide)); diff --git a/types/logic/communication/NetworkCacheManager.d.ts b/types/logic/communication/NetworkCacheManager.d.ts new file mode 100644 index 00000000..b69e7ca1 --- /dev/null +++ b/types/logic/communication/NetworkCacheManager.d.ts @@ -0,0 +1,24 @@ +/** + * This is a specific type of request cache that can use an IndexedDB-backed request / response cache in order to speed + * reanalysing assays that have, for example, failed before. We take into account the maximum amount of storage + * space that can be used by the cache and make sure that this does not overflow. + * + * @author Pieter Verschaffelt + */ +export default class RequestCacheNetworkManager { + private readonly baseUrl; + private readonly cacheKey; + private indexedDb; + private uniprotVersion; + private uniprotVersionLastChecked; + private static readonly MAX_REQUEST_CACHE_SIZE; + private static readonly UNIPROT_VERSION_INVALIDATE_MS; + constructor(baseUrl: string, cacheKey?: string); + postJSON(url: string, data: any): Promise; + getJSON(url: string): Promise; + private setupDb; + private writeRequestToDb; + private readRequestFromDb; + private getEstimatedQuota; + private getUniprotDBVersion; +} diff --git a/types/logic/communication/peptide/Pept2DataCommunicator.d.ts b/types/logic/communication/peptide/Pept2DataCommunicator.d.ts index c4ea91b4..da750e94 100644 --- a/types/logic/communication/peptide/Pept2DataCommunicator.d.ts +++ b/types/logic/communication/peptide/Pept2DataCommunicator.d.ts @@ -9,8 +9,9 @@ export default class Pept2DataCommunicator { private readonly peptdataBatchSize; private readonly missedCleavageBatchSize; private readonly parallelRequests; + readonly cacheKey: string; private cancelled; - constructor(apiBaseUrl?: string, peptdataBatchSize?: number, missedCleavageBatchSize?: number, parallelRequests?: number); + constructor(apiBaseUrl?: string, peptdataBatchSize?: number, missedCleavageBatchSize?: number, parallelRequests?: number, cacheKey?: string); process(countTable: CountTable, enableMissingCleavageHandling: boolean, equateIl: boolean, progressListener?: ProgressListener): Promise<[ShareableMap, PeptideTrust]>; cancel(): void; }