From 5c01a7e19edc43c96131618c33aaaa2c78edc3cb Mon Sep 17 00:00:00 2001 From: Ieben Smessaert Date: Mon, 14 Oct 2024 13:30:22 +0200 Subject: [PATCH] Refactor structure and add Redis support (#29) * refactor: Move MongoDB specific code to its dedicated repository class * feat: Add Redis support * 0.1.0-alpha.4 * fix: No mGet for zero members and escape more chars * 0.1.0-alpha.9 --- README.md | 1 - package-lock.json | 87 ++++++- package.json | 5 +- server/examples/config-ldes-acl.json | 4 +- server/examples/config-ldes-all.json | 173 ------------- server/examples/config-ldes-bluebike.json | 2 +- server/examples/config-ldes-file_locks.json | 4 +- server/examples/config-ldes-rinf.json | 2 +- server/examples/config-ldes-ts.json | 142 ----------- server/examples/config-ldes.json | 4 +- src/{mongoDB/MongoDBConfig.ts => DBConfig.ts} | 12 - src/index.ts | 15 +- src/ldes/SDSFragment.ts | 35 +++ src/ldes/SDSView.ts | 151 +++++++++++ .../viewDescription/MongoTSViewDescription.ts | 77 ------ src/ldes/viewDescription/ViewDescription.ts | 41 +-- .../viewDescription/ViewDescriptionParser.ts | 4 +- src/mongoDB/MongoCollectionTypes.ts | 31 --- src/mongoDB/MongoSDS.ts | 240 ------------------ src/mongoDB/MongoTS.ts | 179 ------------- src/repositories/MongoDBRepository.ts | 85 +++++++ src/repositories/RedisRepository.ts | 124 +++++++++ src/repositories/Repository.ts | 60 +++++ 23 files changed, 585 insertions(+), 893 deletions(-) delete mode 100644 server/examples/config-ldes-all.json delete mode 100644 server/examples/config-ldes-ts.json rename src/{mongoDB/MongoDBConfig.ts => DBConfig.ts} (60%) create mode 100644 src/ldes/SDSFragment.ts create mode 100644 src/ldes/SDSView.ts delete mode 100644 src/ldes/viewDescription/MongoTSViewDescription.ts delete mode 100644 src/mongoDB/MongoCollectionTypes.ts delete mode 100644 src/mongoDB/MongoSDS.ts delete mode 100644 src/mongoDB/MongoTS.ts create mode 100644 src/repositories/MongoDBRepository.ts create mode 100644 src/repositories/RedisRepository.ts create mode 100644 src/repositories/Repository.ts diff --git a/README.md b/README.md index 62c91b4..db428ed 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,6 @@ You will probably want to configure a `urn:solid-server:default:LDESConfig` and - `value` of the relation - `bucket` is the target fragment id - `path` of the relation - - optional `timeStamp` specifies the starting timestamp of this fragment. - `metaCollection` specifies the metadata collection, this collection contains information about each ingested stream. JSON objects with following fields: - `id` the id of the ingested stream. - `type` specifies the metadata type. Often "https://w3id.org/sds#Stream". diff --git a/package-lock.json b/package-lock.json index d02621e..b9b7b1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,18 @@ { "name": "ldes-solid-server", - "version": "0.1.0-alpha.2", + "version": "0.1.0-alpha.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ldes-solid-server", - "version": "0.1.0-alpha.2", + "version": "0.1.0-alpha.9", "dependencies": { "@rdfjs/types": "^1.1.0", "@treecg/types": "^0.4.6", "mongodb": "^6.8.0", - "n3": "^1.21.0" + "n3": "^1.21.0", + "redis": "^4.7.0" }, "devDependencies": { "@solid/community-server": "^7.1.2", @@ -4041,6 +4042,64 @@ "@types/node": "*" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.0.tgz", + "integrity": "sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/client/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz", + "integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz", + "integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz", + "integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@rubensworks/saxes": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/@rubensworks/saxes/-/saxes-6.0.1.tgz", @@ -5863,7 +5922,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=0.10.0" @@ -7255,6 +7313,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10901,6 +10967,19 @@ "dev": true, "license": "MIT" }, + "node_modules/redis": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.7.0.tgz", + "integrity": "sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.6.0", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.7", + "@redis/search": "1.2.0", + "@redis/time-series": "1.1.0" + } + }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", diff --git a/package.json b/package.json index c61b5e1..182e21f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ldes-solid-server", - "version": "0.1.0-alpha.2", + "version": "0.1.0-alpha.9", "types": "./dist/index.d.ts", "main": "./dist/index.js", "files": [ @@ -33,7 +33,8 @@ "@rdfjs/types": "^1.1.0", "@treecg/types": "^0.4.6", "mongodb": "^6.8.0", - "n3": "^1.21.0" + "n3": "^1.21.0", + "redis": "^4.7.0" }, "devDependencies": { "@solid/community-server": "^7.1.2", diff --git a/server/examples/config-ldes-acl.json b/server/examples/config-ldes-acl.json index 4d550ac..be27c3c 100644 --- a/server/examples/config-ldes-acl.json +++ b/server/examples/config-ldes-acl.json @@ -80,7 +80,7 @@ "@type": "PrefixView", "prefix": "example", "view": { - "@type": "MongoTSView", + "@type": "SDSView", "streamId": "http://example.org/myStream#eventStream", "descriptionId": "http://example.org/myStream#viewDescription", "db": { @@ -92,7 +92,7 @@ "@type": "PrefixView", "prefix": "ais", "view": { - "@type": "MongoTSView", + "@type": "SDSView", "streamId": "http://example.org/aisStream#eventStream", "descriptionId": "http://example.org/aisStream#description", "db": { diff --git a/server/examples/config-ldes-all.json b/server/examples/config-ldes-all.json deleted file mode 100644 index c97e3b0..0000000 --- a/server/examples/config-ldes-all.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "@context": [ - "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld", - "https://linkedsoftwaredependencies.org/bundles/npm/ldes-solid-server/^0.0.0/components/context.jsonld" - ], - "import": [ - "css:config/app/init/initialize-intro.json", - "css:config/app/main/default.json", - "css:config/app/variables/default.json", - "css:config/http/handler/default.json", - "css:config/http/middleware/default.json", - "css:config/http/notifications/all.json", - "css:config/http/server-factory/http.json", - "css:config/http/static/default.json", - "css:config/identity/access/public.json", - "css:config/identity/email/default.json", - "css:config/identity/handler/default.json", - "css:config/identity/oidc/default.json", - "css:config/identity/ownership/token.json", - "css:config/identity/pod/static.json", - "css:config/ldp/authentication/dpop-bearer.json", - "css:config/ldp/authorization/allow-all.json", - "lss:config/ldp/handler/default.json", - "css:config/ldp/metadata-parser/default.json", - "lss:config/ldp/metadata-writer/default.json", - "css:config/ldp/modes/default.json", - "css:config/storage/backend/data-accessors/memory.json", - "css:config/storage/key-value/memory.json", - "css:config/storage/location/root.json", - "css:config/storage/middleware/default.json", - "css:config/util/auxiliary/acl.json", - "css:config/util/identifiers/suffix.json", - "css:config/util/index/default.json", - "css:config/util/logging/winston.json", - "css:config/util/representation-conversion/default.json", - "css:config/util/resource-locker/memory.json", - "css:config/util/variables/default.json" - ], - "@graph": [ - { - "comment": "A more complex example with 3 different stores being routed to.", - "@id": "urn:solid-server:default:ResourceStore_Backend", - "@type": "RoutingResourceStore", - "rule": { - "@id": "urn:solid-server:default:RouterRule" - } - }, - { - "@id": "urn:solid-server:default:RouterRule", - "@type": "RegexRouterRule", - "base": { - "@id": "urn:solid-server:default:variable:baseUrl" - }, - "rules": [ - { - "@type": "RegexRule", - "regex": "^/(\\.acl|\\.meta)?$", - "store": { - "@id": "urn:solid-server:default:MemoryResourceStore" - } - }, - { - "@type": "RegexRule", - "regex": { - "@id": "urn:solid-server:default:relative-path" - }, - "store": { - "comment": "A more complex example with 3 different stores being routed to.", - "@type": "RepresentationConvertingStore", - "source": { - "@type": "LDESStore", - "id": "http://mine.org/testing", - "base": { - "@id": "urn:solid-server:default:variable:baseUrl" - }, - "relativePath": { - "@id": "urn:solid-server:default:relative-path" - }, - "views": [ - { - "@type": "PrefixView", - "prefix": "default", - "view": { - "@type": "MongoSDSView", - "descriptionId": "http://localhost:3000/ldes/#timestampFragmentation", - "streamId": "http://me#csvStream", - "db": { - "@id": "urn:solid-server:default:OriginalMongoDBConfig" - } - } - }, - { - "@type": "PrefixView", - "prefix": "mine", - "view": { - "@type": "MongoSDSView", - "streamId": "http://mine.org/rdfstream", - "db": { - "@id": "urn:solid-server:default:OriginalMongoDBConfig" - } - } - }, - { - "@type": "PrefixView", - "prefix": "example", - "view": { - "@type": "MongoTSView", - "streamId": "http://example.org/myStream#eventStream", - "descriptionId": "http://example.org/myStream#viewDescription", - "db": { - "@id": "urn:solid-server:default:DBConfig" - } - } - }, - { - "@type": "PrefixView", - "prefix": "ais", - "view": { - "@type": "MongoTSView", - "streamId": "http://example.org/aisStream#eventStream", - "descriptionId": "http://example.org/aisStream#description", - "db": { - "@id": "urn:solid-server:default:DBConfig" - } - } - } - ] - }, - "options_outConverter": { - "@id": "urn:solid-server:default:RepresentationConverter" - } - } - } - ] - }, - { - "@id": "urn:solid-server:default:MemoryResourceStore", - "@type": "DataAccessorBasedStore", - "identifierStrategy": { - "@id": "urn:solid-server:default:IdentifierStrategy" - }, - "auxiliaryStrategy": { - "@id": "urn:solid-server:default:AuxiliaryStrategy" - }, - "accessor": { - "@id": "urn:solid-server:default:MemoryDataAccessor" - }, - "metadataStrategy": { - "@id": "urn:solid-server:default:MetadataStrategy" - } - }, - { - "@id": "urn:solid-server:default:DBConfig", - "@type": "DBConfig", - "metaCollection": "meta", - "indexCollection": "index", - "membersCollection": "data", - "dbUrl": "mongodb://localhost:27017/ldes" - }, - { - "@id": "urn:solid-server:default:OriginalMongoDBConfig", - "@type": "DBConfig", - "metaCollection": "meta", - "indexCollection": "index", - "membersCollection": "data", - "dbUrl": "mongodb://localhost:27017/ldes2" - }, - { - "@id": "urn:solid-server:default:relative-path", - "valueRaw": "/ldes/" - } - ] -} diff --git a/server/examples/config-ldes-bluebike.json b/server/examples/config-ldes-bluebike.json index 583d63c..b97598d 100644 --- a/server/examples/config-ldes-bluebike.json +++ b/server/examples/config-ldes-bluebike.json @@ -82,7 +82,7 @@ "@type": "PrefixView", "prefix": "default", "view": { - "@type": "MongoSDSView", + "@type": "SDSView", "descriptionId": "http://localhost:3000/ldes/#timestampFragmentation", "streamId": "https://w3id.org/sds#Stream", "db": { diff --git a/server/examples/config-ldes-file_locks.json b/server/examples/config-ldes-file_locks.json index 396ecdb..280abad 100644 --- a/server/examples/config-ldes-file_locks.json +++ b/server/examples/config-ldes-file_locks.json @@ -81,7 +81,7 @@ "@type": "PrefixView", "prefix": "default", "view": { - "@type": "MongoSDSView", + "@type": "SDSView", "descriptionId": "http://localhost:3000/ldes/#timestampFragmentation", "streamId": "http://me#csvStream", "db": { @@ -93,7 +93,7 @@ "@type": "PrefixView", "prefix": "mine", "view": { - "@type": "MongoSDSView", + "@type": "SDSView", "streamId": "http://mine.org/rdfstream", "db": { "@id": "urn:solid-server:default:DBConfig" diff --git a/server/examples/config-ldes-rinf.json b/server/examples/config-ldes-rinf.json index 723c8c2..5ea3056 100644 --- a/server/examples/config-ldes-rinf.json +++ b/server/examples/config-ldes-rinf.json @@ -82,7 +82,7 @@ "@type": "PrefixView", "prefix": "ldes", "view": { - "@type": "MongoSDSView", + "@type": "SDSView", "descriptionId": "http://localhost:3000/rinf/ldes/#timestampFragmentation", "streamId": "https://era.ilabt.imec.be/ldes/metadata#generation-stream", "db": { diff --git a/server/examples/config-ldes-ts.json b/server/examples/config-ldes-ts.json deleted file mode 100644 index 72ffd74..0000000 --- a/server/examples/config-ldes-ts.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "@context": [ - "https://linkedsoftwaredependencies.org/bundles/npm/@solid/community-server/^7.0.0/components/context.jsonld", - "https://linkedsoftwaredependencies.org/bundles/npm/ldes-solid-server/^0.0.0/components/context.jsonld" - ], - "import": [ - "css:config/app/init/initialize-intro.json", - "css:config/app/main/default.json", - "css:config/app/variables/default.json", - "css:config/http/handler/default.json", - "css:config/http/middleware/default.json", - "css:config/http/notifications/all.json", - "css:config/http/server-factory/http.json", - "css:config/http/static/default.json", - "css:config/identity/access/public.json", - "css:config/identity/email/default.json", - "css:config/identity/handler/default.json", - "css:config/identity/oidc/default.json", - "css:config/identity/ownership/token.json", - "css:config/identity/pod/static.json", - "css:config/ldp/authentication/dpop-bearer.json", - "css:config/ldp/authorization/allow-all.json", - "lss:config/ldp/handler/default.json", - "css:config/ldp/metadata-parser/default.json", - "lss:config/ldp/metadata-writer/default.json", - "css:config/ldp/modes/default.json", - "css:config/storage/backend/data-accessors/memory.json", - "css:config/storage/key-value/memory.json", - "css:config/storage/location/root.json", - "css:config/storage/middleware/default.json", - "css:config/util/auxiliary/acl.json", - "css:config/util/identifiers/suffix.json", - "css:config/util/index/default.json", - "css:config/util/logging/winston.json", - "css:config/util/representation-conversion/default.json", - "css:config/util/resource-locker/memory.json", - "css:config/util/variables/default.json" - ], - "@graph": [ - { - "comment": "A more complex example with 3 different stores being routed to.", - "@id": "urn:solid-server:default:ResourceStore_Backend", - "@type": "RoutingResourceStore", - "rule": { - "@id": "urn:solid-server:default:RouterRule" - } - }, - { - "@id": "urn:solid-server:default:RouterRule", - "@type": "RegexRouterRule", - "base": { - "@id": "urn:solid-server:default:variable:baseUrl" - }, - "rules": [ - { - "@type": "RegexRule", - "regex": "^/(\\.acl|\\.meta)?$", - "store": { - "@id": "urn:solid-server:default:MemoryResourceStore" - } - }, - { - "@type": "RegexRule", - "regex": { - "@id": "urn:solid-server:default:relative-path" - }, - "store": { - "comment": "A more complex example with 3 different stores being routed to.", - "@type": "RepresentationConvertingStore", - "source": { - "@type": "LDESStore", - "id": "http://mine.org/testing", - "base": { - "@id": "urn:solid-server:default:variable:baseUrl" - }, - "relativePath": { - "@id": "urn:solid-server:default:relative-path" - }, - "views": [ - { - "@type": "PrefixView", - "prefix": "example", - "view": { - "@type": "MongoTSView", - "streamId": "http://example.org/myStream#eventStream", - "descriptionId": "http://example.org/myStream#viewDescription", - "db": { - "@id": "urn:solid-server:default:DBConfig" - } - } - }, - { - "@type": "PrefixView", - "prefix": "ais", - "view": { - "@type": "MongoTSView", - "streamId": "http://example.org/aisStream#eventStream", - "descriptionId": "http://example.org/aisStream#description", - "db": { - "@id": "urn:solid-server:default:DBConfig" - } - } - } - ] - }, - "options_outConverter": { - "@id": "urn:solid-server:default:RepresentationConverter" - } - } - } - ] - }, - { - "@id": "urn:solid-server:default:MemoryResourceStore", - "@type": "DataAccessorBasedStore", - "identifierStrategy": { - "@id": "urn:solid-server:default:IdentifierStrategy" - }, - "auxiliaryStrategy": { - "@id": "urn:solid-server:default:AuxiliaryStrategy" - }, - "accessor": { - "@id": "urn:solid-server:default:MemoryDataAccessor" - }, - "metadataStrategy": { - "@id": "urn:solid-server:default:MetadataStrategy" - } - }, - { - "@id": "urn:solid-server:default:DBConfig", - "@type": "DBConfig", - "metaCollection": "meta", - "indexCollection": "index", - "membersCollection": "data", - "dbUrl": "mongodb://localhost:27017/ldes" - }, - { - "@id": "urn:solid-server:default:relative-path", - "valueRaw": "/ldes/" - } - ] -} diff --git a/server/examples/config-ldes.json b/server/examples/config-ldes.json index 9e38f8f..20df5ca 100644 --- a/server/examples/config-ldes.json +++ b/server/examples/config-ldes.json @@ -82,7 +82,7 @@ "@type": "PrefixView", "prefix": "default", "view": { - "@type": "MongoSDSView", + "@type": "SDSView", "descriptionId": "http://localhost:3000/ldes/#timestampFragmentation", "streamId": "https://w3id.org/sds#Stream", "db": { @@ -94,7 +94,7 @@ "@type": "PrefixView", "prefix": "indexed", "view": { - "@type": "MongoSDSView", + "@type": "SDSView", "descriptionId": "http://localhost:3000/ldes/#timestampFragmentation", "streamId": "https://example.org/ns#", "db": { diff --git a/src/mongoDB/MongoDBConfig.ts b/src/DBConfig.ts similarity index 60% rename from src/mongoDB/MongoDBConfig.ts rename to src/DBConfig.ts index fe9185c..d8a6e45 100644 --- a/src/mongoDB/MongoDBConfig.ts +++ b/src/DBConfig.ts @@ -1,14 +1,9 @@ -import { Db, MongoClient } from "mongodb"; - export class DBConfig { readonly url: string; readonly meta: string; readonly data: string; readonly index: string; - _client: MongoClient; - _clientInit: unknown; - constructor( metaCollection: string, membersCollection: string, @@ -20,12 +15,5 @@ export class DBConfig { this.index = indexCollection; this.url = dbUrl || "mongodb://localhost:27017/ldes"; - this._client = new MongoClient(this.url); - this._clientInit = this._client.connect(); - } - - async db(): Promise { - await this._clientInit; - return this._client.db(); } } diff --git a/src/index.ts b/src/index.ts index 7ff5774..a7e8ff8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,15 +1,18 @@ -// Database connections (implement the LDES interfaces) -export * from "./mongoDB/MongoSDS"; -export * from "./mongoDB/MongoTS"; -export * from "./mongoDB/MongoDBConfig"; -export * from "./mongoDB/MongoCollectionTypes"; +// Database connections +export * from "./DBConfig"; +export * from "./repositories/Repository"; +export * from "./repositories/MongoDBRepository"; +export * from "./repositories/RedisRepository"; + +// LDES interface implementations +export * from "./ldes/SDSView"; +export * from "./ldes/SDSFragment"; // LDES interfaces export * from "./ldes/View"; export * from "./ldes/Fragment"; // View Description -export * from "./ldes/viewDescription/MongoTSViewDescription"; export * from "./ldes/viewDescription/ViewDescription"; export * from "./ldes/viewDescription/ViewDescriptionParser"; // LDES store diff --git a/src/ldes/SDSFragment.ts b/src/ldes/SDSFragment.ts new file mode 100644 index 0000000..7432fe9 --- /dev/null +++ b/src/ldes/SDSFragment.ts @@ -0,0 +1,35 @@ +import { CacheDirectives, Member } from "@treecg/types"; +import { Fragment, RelationParameters } from "./Fragment"; +import { Repository } from "../repositories/Repository"; + +export class SDSFragment implements Fragment { + members: string[]; + relations: RelationParameters[]; + repository: Repository; + + cacheDirectives: CacheDirectives; + + constructor( + members: string[], + relations: RelationParameters[], + repository: Repository, + cacheDirectives: CacheDirectives, + ) { + this.repository = repository; + this.members = members; + this.relations = relations; + this.cacheDirectives = cacheDirectives; + } + + async getMembers(): Promise { + return await this.repository.findMembers(this.members); + } + + async getRelations(): Promise { + return this.relations; + } + + async getCacheDirectives(): Promise { + return this.cacheDirectives; + } +} diff --git a/src/ldes/SDSView.ts b/src/ldes/SDSView.ts new file mode 100644 index 0000000..74e1911 --- /dev/null +++ b/src/ldes/SDSView.ts @@ -0,0 +1,151 @@ +import { View } from "./View"; +import { getLoggerFor, RedirectHttpError } from "@solid/community-server"; +import { DBConfig } from "../DBConfig"; +import type { Quad, Quad_Object } from "@rdfjs/types"; +import { CacheDirectives, LDES, RDF, RelationType, SDS, TREE } from "@treecg/types"; +import { DataFactory, Parser } from "n3"; +import { Fragment, parseRdfThing, RelationParameters } from "./Fragment"; +import { getRepository, Repository } from "../repositories/Repository"; +import { DCAT } from "../util/Vocabulary"; +import { Parsed, parseIndex, reconstructIndex } from "../util/utils"; +import { SDSFragment } from "./SDSFragment"; +import quad = DataFactory.quad; +import namedNode = DataFactory.namedNode; +import blankNode = DataFactory.blankNode; + +export class SDSView implements View { + dbConfig: DBConfig; + repository: Repository; + roots!: string[]; + descriptionId: string; + + streamId: string; + freshDuration: number = 60; + protected readonly logger = getLoggerFor(this); + + constructor(db: DBConfig, streamId: string, descriptionId: string) { + this.dbConfig = db; + this.repository = getRepository(this.dbConfig); + this.streamId = streamId; + this.descriptionId = descriptionId; + this.roots = []; + } + + async init(base: string, prefix: string, freshDuration: number): Promise { + this.freshDuration = freshDuration; + await this.repository.open(); + + const roots = await this.repository.findRoots(this.streamId); + if (roots.length > 0) { + for (const root of roots) { + this.roots.push( + [ + base.replace(/^\/|\/$/g, ""), + prefix.replace(/^\/|\/$/g, ""), + root, + ].join("/"), + ); + } + } else { + this.roots = [ + [ + base.replace(/^\/|\/$/g, ""), + prefix.replace(/^\/|\/$/g, ""), + ].join("/"), + ]; + } + } + + getCacheDirectives(isImmutable?: boolean): CacheDirectives { + const immutable = !!isImmutable; + const maxAge = immutable ? 604800 : this.freshDuration; + return { + pub: true, + immutable: immutable, + maxAge, + }; + } + + getRoots(): string[] { + return this.roots; + } + + async getMetadata(ldes: string): Promise<[Quad[], Quad_Object]> { + const quads = []; + const blankId = this.descriptionId + ? namedNode(this.descriptionId) + : blankNode(); + for (const root of this.getRoots()) { + quads.push( + quad( + blankId, + RDF.terms.type, + TREE.terms.custom("ViewDescription"), + ), + quad(blankId, DCAT.terms.endpointURL, namedNode(root)), + quad(blankId, DCAT.terms.servesDataset, namedNode(ldes)), + ); + } + + const stream = await this.repository.findMetadata(SDS.Stream, this.streamId); + if (stream) { + quads.push( + quad( + blankId, + LDES.terms.custom("managedBy"), + namedNode(this.streamId), + ), + ); + + quads.push(...new Parser().parse(stream)); + } + + return [quads, blankId]; + } + + async getFragment(identifier: string): Promise { + const { segs, query } = parseIndex(identifier); + const members: string[] = []; + const relations = []; + + const id = segs.length > 0 ? segs.join("/") : (await this.repository.findRoots(this.streamId))[0]; + + this.logger.verbose(`Finding fragment for stream '${this.streamId}' and id '${id}'.`); + const fragment = await this.repository.findBucket(this.streamId, id); + + if (fragment) { + fragment.relations = fragment.relations || []; + + const rels: RelationParameters[] = fragment!.relations.map( + ({ type, value, bucket, path }) => { + const index: Parsed = { segs: bucket.split("/"), query }; + const relation: RelationParameters = { + type: type, + nodeId: reconstructIndex(index), + }; + + if (value) { + relation.value = parseRdfThing(value); + } + if (path) { + relation.path = parseRdfThing(path); + } + + return relation; + }, + ); + relations.push(...rels); + members.push(...(fragment.members || [])); + } else { + this.logger.error(`No such bucket found! (stream: '${this.streamId}', id: '${id}')`); + throw new RedirectHttpError(404, "No fragment found", ""); + } + + return new SDSFragment( + members, + relations, + this.repository, + this.getCacheDirectives(fragment?.immutable), + ); + } +} diff --git a/src/ldes/viewDescription/MongoTSViewDescription.ts b/src/ldes/viewDescription/MongoTSViewDescription.ts deleted file mode 100644 index d70b70d..0000000 --- a/src/ldes/viewDescription/MongoTSViewDescription.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Store } from "n3"; -import { - BucketizeStrategy, - IngestorClient, - IViewDescription, - ViewDescription, -} from "./ViewDescription"; -import { ViewDescriptionParser } from "./ViewDescriptionParser"; - -export class MongoTSViewDescription { - private viewDescriptionIdentifier: string; - private ldesIdentifier: string; - private viewIdentifier: string; - - constructor( - viewDescriptionIdentifier: string, - ldesIdentifier: string, - viewIdentifier?: string, - ) { - this.viewDescriptionIdentifier = viewDescriptionIdentifier; - this.ldesIdentifier = ldesIdentifier; - this.viewIdentifier = viewIdentifier || "dummy"; - } - - public parseViewDescription(store: Store): IViewDescription { - const parser = new ViewDescriptionParser( - this.viewIdentifier, - this.ldesIdentifier, - ); - return parser.parseViewDescription( - store, - this.viewDescriptionIdentifier, - ); - } - - public generateViewDescription(options: { - timestampPath: string; - pageSize?: number; - }): IViewDescription { - const ingestorClientType = - "http://www.example.org/ldes#mongoDBTSIngestor"; - const bucketizationType = "https://w3id.org/ldes#LDESTSFragmentation"; - - const bucketStrategy = new BucketizeStrategy( - this.bucketID(), - bucketizationType, - options.timestampPath, - options.pageSize, - ); - const ingestorClient = new IngestorClient( - this.ingestorID(), - bucketStrategy, - ingestorClientType, - ); - // eventStreamIdentifier and rootNodeIdentifier are not important for generating this viewDescription - // as this is normally added within the LDES solid server. - return new ViewDescription( - this.viewDescriptionIdentifier, - ingestorClient, - this.ldesIdentifier, - "dummy", - ); - } - - private viewDescriptionNamespace(): string { - const viewDescriptionURL = new URL(this.viewDescriptionIdentifier); - return viewDescriptionURL.origin + viewDescriptionURL.pathname + "#"; - } - - private bucketID(): string { - return this.viewDescriptionNamespace() + "bucketizationStrategy"; - } - - private ingestorID(): string { - return this.viewDescriptionNamespace() + "ingestor"; - } -} diff --git a/src/ldes/viewDescription/ViewDescription.ts b/src/ldes/viewDescription/ViewDescription.ts index 269adc9..92096ca 100644 --- a/src/ldes/viewDescription/ViewDescription.ts +++ b/src/ldes/viewDescription/ViewDescription.ts @@ -1,10 +1,10 @@ import { DataFactory, Store } from "n3"; -import namedNode = DataFactory.namedNode; import { LDES, TREE } from "@treecg/types"; import { RDF } from "@solid/community-server"; import { DCAT } from "../../util/Vocabulary"; -import literal = DataFactory.literal; import type * as Rdf from "@rdfjs/types"; +import namedNode = DataFactory.namedNode; +import literal = DataFactory.literal; export interface N3Support { getStore: () => Store; @@ -64,11 +64,6 @@ export interface IBucketizeStrategy extends N3Support { } export class ViewDescription implements IViewDescription { - private _id: string; - private _managedBy: IIngestorClient; - private _servesDataset: string; - private _endpointURL: string; - constructor( id: string, managedBy: IIngestorClient, @@ -81,18 +76,26 @@ export class ViewDescription implements IViewDescription { this._endpointURL = rootNodeIdentifier; } + private _id: string; + get id(): string { return this._id; } + private _managedBy: IIngestorClient; + get managedBy(): IIngestorClient { return this._managedBy; } + private _servesDataset: string; + get servesDataset(): string { return this._servesDataset; } + private _endpointURL: string; + get endpointURL(): string { return this._endpointURL; } @@ -132,11 +135,6 @@ export class ViewDescription implements IViewDescription { } export class IngestorClient implements IIngestorClient { - private _bucketizeStrategy: IBucketizeStrategy; - private _id: string; - - private _type: string; - constructor( id: string, bucketizeStrategy: IBucketizeStrategy, @@ -147,14 +145,20 @@ export class IngestorClient implements IIngestorClient { this._type = type; } + private _bucketizeStrategy: IBucketizeStrategy; + get bucketizeStrategy(): IBucketizeStrategy { return this._bucketizeStrategy; } + private _id: string; + get id(): string { return this._id; } + private _type: string; + get type(): string { return this._type; } @@ -176,11 +180,6 @@ export class IngestorClient implements IIngestorClient { } export class BucketizeStrategy implements IBucketizeStrategy { - private _bucketType: string; - private _id: string; - private _pageSize: number | undefined; - private _path: string; - constructor( id: string, bucketType: string, @@ -193,18 +192,26 @@ export class BucketizeStrategy implements IBucketizeStrategy { this._path = path; } + private _bucketType: string; + get bucketType(): string { return this._bucketType; } + private _id: string; + get id(): string { return this._id; } + private _pageSize: number | undefined; + get pageSize(): number | undefined { return this._pageSize; } + private _path: string; + get path(): string { return this._path; } diff --git a/src/ldes/viewDescription/ViewDescriptionParser.ts b/src/ldes/viewDescription/ViewDescriptionParser.ts index fa8d1a3..bbc5953 100644 --- a/src/ldes/viewDescription/ViewDescriptionParser.ts +++ b/src/ldes/viewDescription/ViewDescriptionParser.ts @@ -5,7 +5,7 @@ import { IViewDescription, ViewDescription, } from "./ViewDescription"; -import { DataFactory, Literal, Parser, Store, Writer } from "n3"; +import { DataFactory, Literal, Store } from "n3"; import { LDES, RDF, TREE } from "@treecg/types"; import * as Rdf from "@rdfjs/types"; import namedNode = DataFactory.namedNode; @@ -18,6 +18,7 @@ export class ViewDescriptionParser { this.viewIdentifier = viewIdentifier; this.ldesIdentifier = ldesIdentifier; } + /** * Parses a selection of an N3 Store to a {@link IViewDescription}. * @@ -77,6 +78,7 @@ export class ViewDescriptionParser { this.viewIdentifier, ); } + protected parseBucketizeStrategy( store: Store, bucketizeStrategyNode: Rdf.Term, diff --git a/src/mongoDB/MongoCollectionTypes.ts b/src/mongoDB/MongoCollectionTypes.ts deleted file mode 100644 index 3aa0892..0000000 --- a/src/mongoDB/MongoCollectionTypes.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { RelationType } from "@treecg/types"; - -export type DataCollectionDocument = { - id: string; - data: string; - timestamp?: string; -}; - -export type MetaCollectionDocument = { - id: string; - value: string; - type: string; -}; - -export type IndexCollectionDocument = { - id?: string; - streamId: string; - leaf: boolean; - value?: string; - relations: { - type: RelationType; - value: string; - bucket: string; - path: string; - timestampRelation?: boolean; - }[]; - members?: string[]; - count: number; - timeStamp?: Date; - immutable?: boolean; -}; diff --git a/src/mongoDB/MongoSDS.ts b/src/mongoDB/MongoSDS.ts deleted file mode 100644 index 55f1947..0000000 --- a/src/mongoDB/MongoSDS.ts +++ /dev/null @@ -1,240 +0,0 @@ -import type { Quad, Quad_Object } from "@rdfjs/types"; -import { getLoggerFor, RedirectHttpError } from "@solid/community-server"; -import { - CacheDirectives, - LDES, - Member, - RDF, - RelationType, - SDS, - TREE, -} from "@treecg/types"; -import { Collection, Db, Filter } from "mongodb"; -import { DataFactory, Parser } from "n3"; -import { View } from "../ldes/View"; -import { Parsed, parseIndex, reconstructIndex } from "../util/utils"; -import { DBConfig } from "./MongoDBConfig"; -import { - DataCollectionDocument, - IndexCollectionDocument, - MetaCollectionDocument, -} from "./MongoCollectionTypes"; -import { Fragment, parseRdfThing, RelationParameters } from "../ldes/Fragment"; -import { DCAT } from "../util/Vocabulary"; - -const { namedNode, quad, blankNode, literal } = DataFactory; - -class MongoSDSFragment implements Fragment { - members: string[]; - relations: RelationParameters[]; - collection: Collection; - - cacheDirectives: CacheDirectives; - constructor( - members: string[], - relations: RelationParameters[], - collection: Collection, - cacheDirectives: CacheDirectives, - ) { - this.collection = collection; - this.members = members; - this.relations = relations; - this.cacheDirectives = cacheDirectives; - } - - async getMembers(): Promise { - return await this.collection - .find({ id: { $in: this.members } }) - .map((row) => { - return { - id: namedNode(row.id), - quads: new Parser().parse(row.data), - }; - }) - .toArray(); - } - - async getRelations(): Promise { - return this.relations; - } - - async getCacheDirectives(): Promise { - return this.cacheDirectives; - } -} - -export class MongoSDSView implements View { - protected readonly logger = getLoggerFor(this); - - dbConfig: DBConfig; - db!: Db; - metaCollection!: Collection; - indexCollection!: Collection; - dataCollection!: Collection; - roots!: string[]; - - descriptionId?: string; - streamId: string; - - freshDuration: number = 60; - - constructor(db: DBConfig, streamId: string, descriptionId?: string) { - this.dbConfig = db; - this.streamId = streamId; - this.descriptionId = descriptionId; - this.roots = []; - } - - async init( - base: string, - prefix: string, - freshDuration: number, - ): Promise { - this.freshDuration = freshDuration; - this.db = await this.dbConfig.db(); - this.metaCollection = this.db.collection(this.dbConfig.meta); - this.indexCollection = this.db.collection(this.dbConfig.index); - this.dataCollection = this.db.collection(this.dbConfig.data); - - const roots = await this.indexCollection - .find({ root: true, streamId: this.streamId }) - .toArray(); - if (roots.length > 0) { - for (const root of roots) { - this.roots.push( - [ - base.replace(/^\/|\/$/g, ""), - prefix.replace(/^\/|\/$/g, ""), - root.id, - ].join("/"), - ); - } - } else { - this.roots = [ - [ - base.replace(/^\/|\/$/g, ""), - prefix.replace(/^\/|\/$/g, ""), - ].join("/"), - ]; - } - } - - getCacheDirectives(isImmutable?: boolean): CacheDirectives { - const immutable = !!isImmutable; - const maxAge = immutable ? 604800 : this.freshDuration; - return { - pub: true, - immutable: immutable, - maxAge, - }; - } - - getRoots(): string[] { - return this.roots; - } - - async getMetadata(ldes: string): Promise<[Quad[], Quad_Object]> { - const quads = []; - const blankId = this.descriptionId - ? namedNode(this.descriptionId) - : blankNode(); - for (const root of this.getRoots()) { - quads.push( - quad( - blankId, - RDF.terms.type, - TREE.terms.custom("ViewDescription"), - ), - quad(blankId, DCAT.terms.endpointURL, namedNode(root)), - quad(blankId, DCAT.terms.servesDataset, namedNode(ldes)), - ); - } - - const stream = await this.metaCollection.findOne({ - type: SDS.Stream, - id: this.streamId, - }); - if (stream) { - quads.push( - quad( - blankId, - LDES.terms.custom("managedBy"), - namedNode(this.streamId), - ), - ); - - quads.push(...new Parser().parse(stream.value)); - } - - return [quads, blankId]; - } - - async getFragment(identifier: string): Promise { - const { segs, query } = parseIndex(identifier); - const members: string[] = []; - const relations = []; - - const search: Filter = { - streamId: this.streamId, - }; - const id = segs.length > 0 ? segs.join("/") : undefined; - const timestampValue = query["timestamp"]; - - if (id) { - search.id = id; - } - - if (timestampValue) { - search.timeStamp = { $lte: new Date(timestampValue) }; - } - - this.logger.verbose("Finding fragment for " + JSON.stringify(search)); - const fragment = await this.indexCollection - .find(search) - .sort({ timeStamp: -1 }) - .limit(1) - .next(); - - if (fragment) { - if (timestampValue && fragment.timeStamp) { - this.logger.info("Redirect to proper fragment URL"); - // Redirect to the correct resource, we now have the timestamp; - throw new RedirectHttpError(302, "found", `/${fragment.id!}`); - } - fragment.relations = fragment.relations || []; - - const rels: RelationParameters[] = fragment!.relations.map( - ({ type, value, bucket, path }) => { - const index: Parsed = { segs: bucket.split("/"), query }; - const relation: RelationParameters = { - type: type, - nodeId: reconstructIndex(index), - }; - - if (value) { - relation.value = parseRdfThing(value); - } - if (path) { - relation.path = parseRdfThing(path); - } - - return relation; - }, - ); - relations.push(...rels); - members.push(...(fragment.members || [])); - } else { - this.logger.error( - "No such bucket found! " + JSON.stringify(search), - ); - throw new RedirectHttpError(404, "No fragment found", ""); - } - - return new MongoSDSFragment( - members, - relations, - this.dataCollection, - this.getCacheDirectives(fragment?.immutable), - ); - } -} diff --git a/src/mongoDB/MongoTS.ts b/src/mongoDB/MongoTS.ts deleted file mode 100644 index b7c6484..0000000 --- a/src/mongoDB/MongoTS.ts +++ /dev/null @@ -1,179 +0,0 @@ -import type * as Rdf from "@rdfjs/types"; -import { - ensureTrailingSlash, - getLoggerFor, - trimTrailingSlashes, -} from "@solid/community-server"; -import { - CacheDirectives, - LDES, - Member, - RDF, - RelationType, - TREE, -} from "@treecg/types"; -import { Collection, Db, Filter } from "mongodb"; -import { DataFactory, Parser, Store } from "n3"; - -import { View } from "../ldes/View"; -import { Fragment, RdfThing, RelationParameters } from "../ldes/Fragment"; -import { DBConfig } from "./MongoDBConfig"; -import { - DataCollectionDocument, - IndexCollectionDocument, - MetaCollectionDocument, -} from "./MongoCollectionTypes"; -import { MongoTSViewDescription } from "../ldes/viewDescription/MongoTSViewDescription"; - -const { namedNode, quad, literal } = DataFactory; - -class MongoTSFragment implements Fragment { - members: string[]; - relations: RelationParameters[]; - collection: Collection; - - constructor( - members: string[], - relations: RelationParameters[], - collection: Collection, - ) { - this.collection = collection; - this.members = members; - this.relations = relations; - } - - async getMembers(): Promise { - return await this.collection - .find({ id: { $in: this.members } }) - .map((row) => { - return { - id: namedNode(row.id), - quads: new Parser().parse(row.data), - }; - }) - .toArray(); - } - - async getRelations(): Promise { - return this.relations; - } - - async getCacheDirectives(): Promise { - return { pub: true }; - } -} - -export class MongoTSView implements View { - protected readonly logger = getLoggerFor(this); - - dbConfig: DBConfig; - db!: Db; - metaCollection!: Collection; - indexCollection!: Collection; - dataCollection!: Collection; - roots!: string[]; - - descriptionId: string; - streamId: string; - - constructor(db: DBConfig, streamId: string, descriptionId: string) { - this.dbConfig = db; - this.streamId = streamId; - this.descriptionId = descriptionId; - } - - async init(base: string, prefix: string): Promise { - this.db = await this.dbConfig.db(); - this.metaCollection = this.db.collection(this.dbConfig.meta); - this.indexCollection = this.db.collection(this.dbConfig.index); - this.dataCollection = this.db.collection(this.dbConfig.data); - - this.roots = [base + ensureTrailingSlash(trimTrailingSlashes(prefix))]; - } - - /** - * The URL of the view of the LDES. - * @returns {string} - */ - getRoots(): string[] { - return this.roots; - } - - async getMetadata(ldes: string): Promise<[Rdf.Quad[], Rdf.Quad_Object]> { - const quads = []; - const query = { type: LDES.EventStream, id: this.streamId }; - const meta = await this.metaCollection.findOne(query); - if (meta) { - const metaStore = new Store(new Parser().parse(meta.value)); - const mongoTSVD = new MongoTSViewDescription( - this.descriptionId, - ldes, - this.roots[0], - ); - const viewDescription = mongoTSVD.parseViewDescription(metaStore); - quads.push(...viewDescription.quads()); - } else { - this.logger.info("No ViewDescription found for " + this.descriptionId); - } - - quads.push( - quad(namedNode(ldes), RDF.terms.type, LDES.terms.EventStream), - ); - quads.push( - quad( - namedNode(this.getRoots()[0]), - RDF.terms.type, - TREE.terms.custom("Node"), - ), - ); // TODO: verify if this makes sense - return [quads, namedNode(this.descriptionId)]; - } - - async getFragment(identifier: string): Promise { - this.logger.info( - `Looking for fragment with id "${identifier}" in the Mongo Database. (streamID: "${this.streamId}")`, - ); - const members = [] as string[]; - const relations = []; - const search: Filter = { - streamId: this.streamId, - id: identifier, - leaf: true, - }; - - const dbFragment = await this.indexCollection - .find(search) - .sort({ timeStamp: -1 }) - .limit(1) - .next(); - if (!dbFragment) { - this.logger.error( - "No such bucket found! " + JSON.stringify(search), - ); - } else { - members.push(...(dbFragment.members || [])); - - const rels: RelationParameters[] = dbFragment!.relations.map( - ({ type, value, bucket, path }) => { - return { - type: type, - value: { - id: literal( - value, - namedNode( - "http://www.w3.org/2001/XMLSchema#dateTime", - ), - ), - quads: [], - }, - nodeId: bucket, - path: { id: namedNode(path), quads: [] }, - }; - }, - ); - - relations.push(...rels); - } - return new MongoTSFragment(members, relations, this.dataCollection); - } -} diff --git a/src/repositories/MongoDBRepository.ts b/src/repositories/MongoDBRepository.ts new file mode 100644 index 0000000..0d1a1cf --- /dev/null +++ b/src/repositories/MongoDBRepository.ts @@ -0,0 +1,85 @@ +import { Bucket, Repository } from "./Repository"; +import { getLoggerFor } from "@solid/community-server"; +import { Db, MongoClient } from "mongodb"; +import { Member } from "@treecg/types"; +import { DataFactory, Parser } from "n3"; +import namedNode = DataFactory.namedNode; + +export class MongoDBRepository implements Repository { + protected url: string; + protected metadata: string; + protected data: string; + protected index: string; + + protected client: MongoClient | undefined; + protected db: Db | undefined; + + protected logger = getLoggerFor(this); + + constructor(url: string, metadata: string, data: string, index: string) { + this.url = url; + this.metadata = metadata; + this.data = data; + this.index = index; + } + + async open(): Promise { + this.client = await new MongoClient(this.url).connect(); + this.db = this.client.db(); + + this.logger.debug(`Connected to ${this.url}`); + } + + async close(): Promise { + await this.client?.close(); + + this.logger.debug(`Closed connection to ${this.url}`); + } + + async findMetadata(type: string, id: string): Promise { + return this.db?.collection(this.metadata) + .findOne({ type, id }) + .then((entry) => entry?.value ?? null); + } + + async findRoots(streamId: string): Promise { + return this.db?.collection(this.index) + .find({ root: true, streamId }) + .toArray() + .then((roots) => roots.map((root) => root.id)) ?? []; + } + + async findBucket(type: string, id: string): Promise { + const filter: any = { streamId: type, id: id }; + return this.db?.collection(this.index) + .find(filter) + .limit(1) + .next() + .then((entry) => { + if (!entry) { + return null; + } + return { + id: entry.id, + streamId: entry.streamId, + root: entry.root, + value: entry.value, + immutable: entry.immutable, + members: entry.members, + relations: entry.relations, + }; + }) ?? null; + } + + async findMembers(members: string[]): Promise { + return this.db?.collection(this.data) + .find({ id: { $in: members } }) + .map((row) => { + return { + id: namedNode(row.id), + quads: new Parser().parse(row.data), + }; + }) + .toArray() ?? []; + } +} diff --git a/src/repositories/RedisRepository.ts b/src/repositories/RedisRepository.ts new file mode 100644 index 0000000..d1453b7 --- /dev/null +++ b/src/repositories/RedisRepository.ts @@ -0,0 +1,124 @@ +import { Bucket, Repository } from "./Repository"; +import { createClient, RedisClientType, RediSearchSchema, SchemaFieldTypes } from "redis"; +import { getLoggerFor } from "@solid/community-server"; +import { Member } from "@treecg/types"; +import { DataFactory, Parser } from "n3"; +import namedNode = DataFactory.namedNode; + +export class RedisRepository implements Repository { + protected url: string; + protected metadata: string; + protected data: string; + protected index: string; + + protected client: RedisClientType | undefined; + + protected logger = getLoggerFor(this); + + constructor(url: string, metadata: string, data: string, index: string) { + this.url = url; + this.metadata = metadata; + this.data = data; + this.index = index; + } + + async open(): Promise { + this.client = createClient({ url: this.url }); + await this.client.connect(); + + this.logger.debug(`Connected to ${this.url}`); + + await this.createSearchIndex(); + } + + async close(): Promise { + await this.client?.disconnect(); + + this.logger.debug(`Closed connection to ${this.url}`); + } + + async findMetadata(type: string, id: string): Promise { + return this.client?.get(`${this.metadata}:${encodeURIComponent(type)}:${encodeURIComponent(id)}`) ?? null; + } + + async findRoots(streamId: string): Promise { + const result = await this.client?.ft.search(`idx:${this.index}`, `(@streamId:{${this.encodeKey(streamId)}}) (@root:{true})`, { RETURN: ["id"] }); + return result?.documents.map((doc) => doc.value.id as string) ?? []; + } + + async findBucket(type: string, id: string): Promise { + const query = `(@streamId:{${this.encodeKey(type)}) (@id:{${this.encodeKey(id)}})`; + const result = (await this.client?.ft.search(`idx:${this.index}`, query))?.documents ?? []; + const doc = result.pop(); + if (!doc) { + return null; + } + + const members = await this.client?.sMembers(`${this.index}:${encodeURIComponent(type)}:${encodeURIComponent(doc.value.id as string)}:members`) ?? []; + const relations = ((await this.client?.sMembers(`${this.index}:${encodeURIComponent(type)}:${encodeURIComponent(doc.value.id as string)}:relations`)) ?? []).map((relation) => JSON.parse(relation)); + + return { + id: doc.id, + streamId: type, + root: doc.value.root as unknown as boolean, + value: doc.value.value ? (doc.value.value as string) : undefined, + immutable: doc.value.immutable as unknown as boolean, + members: members, + relations: relations, + }; + } + + async findMembers(members: string[]): Promise { + if (members.length === 0) { + return []; + } + return (await this.client?.mGet(members.map((member) => `${this.data}:${encodeURIComponent(member)}`)) ?? []) + .filter((entry) => entry !== null) + .map((entry, i) => { + return { + id: namedNode(members[i]), + quads: new Parser().parse(entry), + }; + }); + } + + async createSearchIndex(): Promise { + try { + const schema: RediSearchSchema = { + "$.streamId": { + type: SchemaFieldTypes.TAG, + AS: "streamId", + }, + "$.id": { + type: SchemaFieldTypes.TAG, + AS: "id", + }, + "$.root": { + type: SchemaFieldTypes.TAG, + AS: "root", + }, + "$.value": { + type: SchemaFieldTypes.TEXT, + AS: "value", + }, + "$.immutable": { + type: SchemaFieldTypes.TAG, + AS: "immutable", + }, + }; + await this.client?.ft.create(`idx:${this.index}`, schema, { + ON: "JSON", + PREFIX: this.index, + }); + this.logger.info(`Created index idx:${this.index}`); + } catch (e) { + // Index already exists + this.logger.debug(`Index idx:${this.index} already exists`); + } + } + + encodeKey(key: string): string { + // Add \\ in front of ., :, /, -, _, % + return key.replace(/([.:\/\-_%])/g, "\\$1"); + } +} diff --git a/src/repositories/Repository.ts b/src/repositories/Repository.ts new file mode 100644 index 0000000..4fb934b --- /dev/null +++ b/src/repositories/Repository.ts @@ -0,0 +1,60 @@ +import { DBConfig } from "../DBConfig"; +import { env } from "process"; +import { MongoDBRepository } from "./MongoDBRepository"; +import { Member } from "@treecg/types"; +import { RedisRepository } from "./RedisRepository"; + +export type Bucket = { + id: string; + streamId: string; + root: boolean; + value?: string; + immutable?: boolean; + members: string[]; + relations: Relation[]; +}; + +export type Relation = { + bucket: string; + path: string; + type: string; + value: string; + timestampRelation?: boolean; +}; + +export interface Repository { + open(): Promise; + + close(): Promise; + + findMetadata(type: string, id: string): Promise; + + findRoots(streamId: string): Promise; + + findBucket(type: string, id: string): Promise; + + findMembers(members: string[]): Promise; +} + +export function getRepository(dbConfig: DBConfig): Repository { + const url = + dbConfig.url || env.DB_CONN_STRING || "mongodb://localhost:27017/ldes"; + + if (url.startsWith("mongodb")) { + return new MongoDBRepository( + url, + dbConfig.meta, + dbConfig.data, + dbConfig.index, + ); + } else if (url.startsWith("redis")) { + return new RedisRepository( + url, + dbConfig.meta, + dbConfig.data, + dbConfig.index, + ); + } else { + throw new Error("Unknown database type"); + } +}