Ldes graphdb connector

README.md

@@ -13,7 +13,7 @@ A Linked Data Event Stream (LDES) is a collection of immutable objects. The HTTP
 ```bash
 npm install
 npm run build
-cd packages/ldes-replicator && npm run start
+npm run start -- -h
 ```
 
 ## Components
@@ -27,6 +27,12 @@ A connector implementation for PostgreSQL.
 **ldes-mongodb-connector:**<br />
 A connector implementation for MongoDB.
 
+**ldes-graphdb-connector:**<br />
+A connector implementation for GraphDB. (can be easily extended to any other datastore compatible with SPARQL)
+
+**ldes-graphdb-version-materialization-connector:**<br />
+Based on the GraphDB connector, for the specific case we only want to keep the last version.
+
 **ldes-dummy-state:**<br />
 An example to write your own state.
 
@@ -60,9 +66,53 @@ Exposes the helm/compose file generators as a Fastify plugin.
 - [Ryan Byloos](https://github.com/ryanbyloos)
 - [Wout Verbiest](https://github.com/woutverbiest)
 
+## Config file
+
+The LDES Replicator is a CLI that takes a JSON file for configuration,
+an example can be found [here](/osoc21/ldes2service/blob/main/config.json.example).
+
+```json
+{
+  // Main settings
+  "replicator": {
+    "ldes": [
+      {
+        "url": "URL to the LDES"
+      },
+      {
+        "url": "URL to the LDES",
+        "shapeUrl": "URL or path of a SHACL shape"
+      }
+    ],
+    "state": {
+      "id": "Used to identify the state in a shared redis server",
+      "host": "hostname",
+      "port": "port number",
+      "password": "password, if needed"
+    },
+    "disable_polling": "If the replicator should stop after the last page of the ldes has been reached",
+    "polling_interval": "delay in ms"
+  },
+  // Connector settings
+  "connectors": {
+    "graphdb": {
+      // NPM package to use as a connector
+      "type": "@ldes/ldes-graphdb-connector",
+      // Connector specific settings cf. Connectors
+      "settings": {
+        ...
+      }
+    }
+  }
+}
+```
+
+Once the config file is created you can add the dynamic dependencies with `npm run start -- -s <config_path>`
+and once it's done, start the LDES replicator with `npm run start -- <config_path>`
+
 ## Running it with Docker
 
-The LDES Replicator is available as a docker image: `ghcr.io/osoc21/ldes-replicator:helm-chart`
+The LDES Replicator is available as a docker image: `ghcr.io/osoc21/ldes-replicator`
 
 ### Requirements
 
@@ -73,15 +123,26 @@ A redis server and a backend server of your choice are needed for the LDES repli
 ```shell
 docker run \
   -e URLS=ldes.example \
-  -e STATE_CONFIG={"host":"127.0.0.1", "id":"replicator"} \
+  -e 'STATE_CONFIG={"host":"127.0.0.1", "id":"replicator"}' \
   -e CONNECTORS=[0] \
   -e CONNECTOR_0_TYPE=ldes-backend-connector \
-  -e CONNECTOR_0_CONFIG={...} \
+  -e 'CONNECTOR_0_CONFIG={...}' \
+  ghcr.io/osoc21/ldes-replicator:latest
+```
+
+or, if a config file is available:
+
+```shell
+docker run \
+  -v $(pwd)/config.json:./config.json
   ghcr.io/osoc21/ldes-replicator:latest
 ```
 
 ### Environment Variables
 
+Environment variables can be used to automatically create a [Config file](#config-file),
+otherwise those can be ignored and a config file mounted at `./config.json`.
+
 - `URLS`: comma-separated list of LDES URLs.
 - `STATE_CONFIG`: JSON object with the Redis connection settings.
 
@@ -102,48 +163,84 @@ docker run \
 - `CONNECTOR_<ID>_TYPE`: Name of the NPM package to use as a backend connector, a list of those can be found below.
 - `CONNECTOR_<ID>_CONFIG`: JSON object with the backend connector settings, those will depend on the connector used.
 
-### Connectors
+## Connectors
 
-#### @ldes/ldes-postgres-connector
+### Generic settings
+
+The connectors will try to implement generic settings to allow a more customizable replication:
+
+```js
+CONFIG:
+{
+  // Handles versioning, a SHACL shape is required. If not provided all the versions will be kept.
+  versions: {
+    // Amount of versions to keep
+    amount: number,
+    // Field of the LDES that identifies versions of the same element
+    identifier: string,
+    // Field of the LDES used to sort the versions (and get rid of the older ones)
+    sorter: string,
+  },
+  "other connector specific settings": ...
+}
+```
+
+### @ldes/ldes-postgres-connector
 
 Stores the Linked Data Events inside a postgres DB backend.
 
 ```js
 CONFIG:
 {
-  //The name of the table the events will be stored in
-  "tableName": string,
   //The Postgres server credentials
   "username": string,
   "password": string,
   "database": string,
   //The Postgres server hostname/IP address and port
   "hostname": string,
   "port": number
-  //How many versions of an LDES element should be kept (0 => any amount)
-  "amountOfVersions": number
 }
 ```
 
-#### @ldes/ldes-mongodb-connector
+### @ldes/ldes-mongodb-connector
 
 Stores the Linked Data Events inside a MongoDB backend.
 
 ```js
 CONFIG:
 {
-  //The MongoDB server credentials
+  //The MongoDB server credentials, if needed
   "username": string,
   "password": string,
   "database": string,
   //The MongoDB server hostname/IP address and port
   "hostname": string,
   "port": number
-  //How many versions of an LDES element should be kept (0 => any amount)
-  "amountOfVersions": number
 }
 ```
 
+### @ldes/ldes-graphdb-connector and @ldes/ldes-graphdb-version-materialization-connector
+
+Stores the Linked Data Events inside a GraphDB backend.
+
+```js
+CONFIG:
+{
+  //The GraphDB server credentials, if needed
+  "password": string,
+  "database": string,
+  //The GraphDB server connection string and the repository to use
+  "baseUrl": string,
+  "repository": number,
+  //Each ldes is stored in a different graph, with this you can provide their URI prefix
+  "graphPrefix": string
+  // Field of the LDES to use as a label, this will create a Lucine connector for full-text search
+  "luceneLabel": string
+}
+```
+
+_Note: The version-materialization connector requires the versions setting (specifically the identifier field)._
+
 ### Docker-compose
 
 Docker compose can be used to host the Redis server, LDES Replicator and on the same dedicated server.
@@ -167,8 +264,6 @@ services:
       CONNECTORS: '[postgres]'
       CONNECTOR_postgres_TYPE: '@ldes/ldes-postgres-connector'
       CONNECTOR_postgres_CONFIG: '{
-        "amountOfVersions":0,
-        "tableName":"ldes",
         "username":"postgres",
         "password":"postgres",
         "database":"postgres",
@@ -212,8 +307,6 @@ releases:
           PG:
             type: '@ldes/ldes-postgres-connector'
             settings:
-              amountOfVersions: 0
-              tableName: 'ldes'
               username: 'postgres'
               password: 'postgres'
               database: 'postgres'

config.json.example

@@ -13,6 +13,7 @@
       "id": "replicator",
       "host": "localhost"
     },
+    "disable_pulling": true,
     "polling_interval": 5000
   },
   "connectors": {

packages/ldes-dummy-connector/lib/DummyConnector.ts

@@ -3,13 +3,6 @@ import type { IConfigConnector, IWritableConnector, LdesShape } from '@ldes/type
 export class DummyConnector implements IWritableConnector {
   private readonly members: any[];
 
-  /**
-   * Templates for the backend generator.
-   */
-  public static composeTemplate = ``;
-
-  public static helmTemplate = ``;
-
   public constructor(config: IConfigConnector, shape: LdesShape, id: string) {
     this.members = [];
     console.log('Shape:');

packages/ldes-graphdb-connector/index.ts

@@ -0,0 +1 @@
+export * from './lib/GraphDbConnector';