From 6df76e1b1db7465a09712fcf637f8bd56be0e69b Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 15:36:16 +0200 Subject: [PATCH 01/14] Add tools as git submodule --- .gitmodules | 3 +++ automatic-submission-flow-tools | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 automatic-submission-flow-tools diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fd994a2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "automatic-submission-flow-tools"] + path = automatic-submission-flow-tools + url = git@github.com:lblod/automatic-submission-flow-tools.git diff --git a/automatic-submission-flow-tools b/automatic-submission-flow-tools new file mode 160000 index 0000000..f4d08e3 --- /dev/null +++ b/automatic-submission-flow-tools @@ -0,0 +1 @@ +Subproject commit f4d08e3639d2d78ffac372ad6200d5e9a601939a From 6dc25f8d9037c46b9f7415e863dfa4728e6c8007 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 15:37:38 +0200 Subject: [PATCH 02/14] Babel config, added packages, fixed submission model Added config for Babel in an attempt to not have it compile the node_modules folder. Added the packages that are dependencies for the tools. The submission model change is unfortunate. This was introduced in the rewrite with the tools which makes the rewritten services somewhat incompatible with the original services. --- .babelrc.json | 9 +++++++++ automatic-submission-flow-tools | 2 +- lib/remote-data-object.js | 2 +- lib/submission.js | 2 +- package.json | 6 ++++++ 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 .babelrc.json diff --git a/.babelrc.json b/.babelrc.json new file mode 100644 index 0000000..dcb79ad --- /dev/null +++ b/.babelrc.json @@ -0,0 +1,9 @@ +{ + "overrides": [ + { "ignore": "./automatic-submission-flow-tools/node_modules" } + ], + "babelrcRoots": [ + ".", + "./packages/*" + ] +} diff --git a/automatic-submission-flow-tools b/automatic-submission-flow-tools index f4d08e3..2718b2f 160000 --- a/automatic-submission-flow-tools +++ b/automatic-submission-flow-tools @@ -1 +1 @@ -Subproject commit f4d08e3639d2d78ffac372ad6200d5e9a601939a +Subproject commit 2718b2ffe336f6f0140bc088857cd1b199faa3a2 diff --git a/lib/remote-data-object.js b/lib/remote-data-object.js index f8ddff6..3027163 100644 --- a/lib/remote-data-object.js +++ b/lib/remote-data-object.js @@ -63,7 +63,7 @@ export class RemoteDataObject { ASK { ?s nie:hasPart ${sparqlEscapeUri(remoteDataObject)}. - ?fo prov:generated ?s. + ?fo prov:generatedBy ?s. ?fo a ; task:operation . diff --git a/lib/submission.js b/lib/submission.js index a053e58..e3f1245 100644 --- a/lib/submission.js +++ b/lib/submission.js @@ -170,7 +170,7 @@ export async function getSubmissionByTask(taskUri) { ${sparqlEscapeUri(taskUri)} a task:Task ; dct:isPartOf ?job . - ?job prov:generated ?submission . + ?job prov:generatedBy ?submission . ?submission dct:subject ?submissionDocument ; adms:status ?status . diff --git a/package.json b/package.json index fccc92c..8ed4b15 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,12 @@ "@lblod/mu-auth-sudo": "^0.6.0", "@lblod/submission-form-helpers": "^1.3.0", "body-parser": "1.20.0", + "automatic-submission-flow-tools": "file:automatic-submission-flow-tools", + "n3": "^1.16.2", + "rdf-string-ttl": "github:benjay10/rdf-string-ttl.js", + "sparqljson-parse": "^2.1.1", + "uuid": "^9.0.0", + "forking-store": "^1.0.0", "fs-extra": "^8.1.0", "lodash.flatten": "^4.4.0", From c770b8f16e3942753b8138766bb05a62fdb35178 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 15:42:12 +0200 Subject: [PATCH 03/14] Cleaned up constants an used errors from tools --- app.js | 41 +++++++++++++++++++++++++++-------------- env.js | 50 -------------------------------------------------- lib/utils.js | 43 ------------------------------------------- 3 files changed, 27 insertions(+), 107 deletions(-) delete mode 100644 lib/utils.js diff --git a/app.js b/app.js index 95c3846..be1d133 100644 --- a/app.js +++ b/app.js @@ -9,7 +9,13 @@ import { CONCEPT_STATUS, } from './lib/submission'; import * as env from './env.js'; -import { saveError } from './lib/utils.js'; +import * as cts from './automatic-submission-flow-tools/constants.js'; +import * as tsk from './automatic-submission-flow-tools/asfTasks.js'; +import * as del from './automatic-submission-flow-tools/deltas.js'; +import * as smt from './automatic-submission-flow-tools/asfSubmissions.js'; +import * as err from './automatic-submission-flow-tools/errors.js'; +import * as N3 from 'n3'; +const { namedNode } = N3.DataFactory; app.use(errorHandler); app.use( @@ -34,17 +40,16 @@ app.post('/delta', async function (req, res) { try { //Don't trust the delta-notifier, filter as best as possible. We just need the task that was created to get started. - const actualTaskUris = req.body - .map((changeset) => changeset.inserts) - .filter((inserts) => inserts.length > 0) - .flat() - .filter((insert) => insert.predicate.value === env.OPERATION_PREDICATE) - .filter((insert) => insert.object.value === env.VALIDATE_OPERATION) - .map((insert) => insert.subject.value); + const actualTasks = del.getSubjects( + req.body, + namedNode(cts.PREDICATE_TABLE.task_operation), + namedNode(cts.OPERATIONS.validate) + ); - for (const taskUri of actualTaskUris) { + for (const task of actualTasks) { + const taskUri = task.value; try { - await updateTaskStatus(taskUri, env.TASK_ONGOING_STATUS); + await updateTaskStatus(taskUri, cts.TASK_STATUSES.busy); const submission = await getSubmissionByTask(taskUri); const { status, logicalFileUri } = await submission.process(); @@ -65,7 +70,7 @@ app.post('/delta', async function (req, res) { await updateTaskStatus( taskUri, - env.TASK_SUCCESS_STATUS, + cts.TASK_STATUSES.success, undefined, //Potential errorURI saveStatus, logicalFileUri @@ -74,8 +79,12 @@ app.post('/delta', async function (req, res) { const message = `Something went wrong while enriching for task ${taskUri}`; console.error(`${message}\n`, error.message); console.error(error); - const errorUri = await saveError({ message, detail: error.message }); - await updateTaskStatus(taskUri, env.TASK_FAILURE_STATUS, errorUri); + const errorUri = await err.create( + namedNode(cts.SERVICES.validateSubmission), + message, + error.message + ); + await updateTaskStatus(taskUri, cts.TASK_STATUSES.failed, errorUri); } } } catch (error) { @@ -83,7 +92,11 @@ app.post('/delta', async function (req, res) { 'The task for enriching a submission could not even be started or finished due to an unexpected problem.'; console.error(`${message}\n`, error.message); console.error(error); - await saveError({ message, detail: error.message }); + await err.create( + namedNode(cts.SERVICES.validateSubmission), + message, + error.message + ); } }); diff --git a/env.js b/env.js index b42cb0b..ea92a1d 100644 --- a/env.js +++ b/env.js @@ -1,54 +1,4 @@ -export const CREATOR = - 'http://lblod.data.gift/services/validate-submission-service'; - -export const TASK_ONGOING_STATUS = - 'http://redpencil.data.gift/id/concept/JobStatus/busy'; -export const TASK_SUCCESS_STATUS = - 'http://redpencil.data.gift/id/concept/JobStatus/success'; -export const TASK_FAILURE_STATUS = - 'http://redpencil.data.gift/id/concept/JobStatus/failed'; - export const TASK_SUCCESSFUL_CONCEPT_STATUS = 'http://lblod.data.gift/automatische-melding-statuses/successful-concept'; export const TASK_SUCCESSFUL_SENT_STATUS = 'http://lblod.data.gift/automatische-melding-statuses/successful-sent'; - -export const OPERATION_PREDICATE = - 'http://redpencil.data.gift/vocabularies/tasks/operation'; -export const VALIDATE_OPERATION = - 'http://lblod.data.gift/id/jobs/concept/TaskOperation/validate'; - -export const PREFIX_TABLE = { - meb: 'http://rdf.myexperiment.org/ontologies/base/', - xsd: 'http://www.w3.org/2001/XMLSchema#', - pav: 'http://purl.org/pav/', - dct: 'http://purl.org/dc/terms/', - melding: 'http://lblod.data.gift/vocabularies/automatische-melding/', - lblodBesluit: 'http://lblod.data.gift/vocabularies/besluit/', - adms: 'http://www.w3.org/ns/adms#', - muAccount: 'http://mu.semte.ch/vocabularies/account/', - eli: 'http://data.europa.eu/eli/ontology#', - org: 'http://www.w3.org/ns/org#', - elod: 'http://linkedeconomy.org/ontology#', - nie: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#', - prov: 'http://www.w3.org/ns/prov#', - mu: 'http://mu.semte.ch/vocabularies/core/', - foaf: 'http://xmlns.com/foaf/0.1/', - nfo: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#', - ext: 'http://mu.semte.ch/vocabularies/ext/', - http: 'http://www.w3.org/2011/http#', - rpioHttp: 'http://redpencil.data.gift/vocabularies/http/', - dgftSec: 'http://lblod.data.gift/vocabularies/security/', - dgftOauth: 'http://kanselarij.vo.data.gift/vocabularies/oauth-2.0-session/', - wotSec: 'https://www.w3.org/2019/wot/security#', - task: 'http://redpencil.data.gift/vocabularies/tasks/', - asj: 'http://data.lblod.info/id/automatic-submission-job/', - dbpedia: 'http://dbpedia.org/ontology/', -}; - -export const PREFIXES = (() => { - const all = []; - for (const key in PREFIX_TABLE) - all.push(`PREFIX ${key}: <${PREFIX_TABLE[key]}>`); - return all.join('\n'); -})(); diff --git a/lib/utils.js b/lib/utils.js deleted file mode 100644 index 3845a79..0000000 --- a/lib/utils.js +++ /dev/null @@ -1,43 +0,0 @@ -import * as mas from '@lblod/mu-auth-sudo'; -import * as mu from 'mu'; -import * as env from '../env.js'; - -export async function saveError({ message, detail, reference }) { - if (!message) throw 'Error needs a message describing what went wrong.'; - const id = mu.uuid(); - const uri = `http://data.lblod.info/errors/${id}`; - const referenceTriple = reference - ? `dct:references ${mu.sparqlEscapeUri(reference)} ;` - : ''; - const detailTriple = detail - ? `oslc:largePreview ${mu.sparqlEscapeString(detail)} ;` - : ''; - const q = ` - PREFIX mu: - PREFIX oslc: - PREFIX dct: - PREFIX xsd: - - INSERT DATA { - GRAPH { - ${mu.sparqlEscapeUri(uri)} - a oslc:Error ; - mu:uuid ${mu.sparqlEscapeString(id)} ; - dct:subject ${mu.sparqlEscapeString('Automatic Submission Service')} ; - oslc:message ${mu.sparqlEscapeString(message)} ; - dct:created ${mu.sparqlEscapeDateTime(new Date().toISOString())} ; - ${referenceTriple} - ${detailTriple} - dct:creator ${mu.sparqlEscapeUri(env.CREATOR)} . - } - } - `; - try { - await mas.updateSudo(q); - return uri; - } catch (e) { - console.warn( - `[WARN] Something went wrong while trying to store an error.\nMessage: ${e}\nQuery: ${q}` - ); - } -} From f6fd066e85814033ec09c91eec1df4a45973b881 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 16:09:55 +0200 Subject: [PATCH 04/14] Rewrite env and constants --- app.js | 17 +++++++---------- env.js | 7 +++++++ lib/file-helpers.js | 12 ++++++------ lib/remote-data-object.js | 4 ++-- lib/submission-task.js | 8 ++++---- lib/submission.js | 16 +++++----------- 6 files changed, 31 insertions(+), 33 deletions(-) diff --git a/app.js b/app.js index be1d133..4007225 100644 --- a/app.js +++ b/app.js @@ -4,9 +4,6 @@ import { updateTaskStatus } from './lib/submission-task'; import { getSubmissionByTask, getSubmissionBySubmissionDocument, - SUBMITABLE_STATUS, - SENT_STATUS, - CONCEPT_STATUS, } from './lib/submission'; import * as env from './env.js'; import * as cts from './automatic-submission-flow-tools/constants.js'; @@ -57,10 +54,10 @@ app.post('/delta', async function (req, res) { let saveStatus; switch (resultingStatus) { - case SENT_STATUS: + case env.SENT_STATUS: saveStatus = env.TASK_SUCCESSFUL_SENT_STATUS; break; - case CONCEPT_STATUS: + case env.CONCEPT_STATUS: saveStatus = env.TASK_SUCCESSFUL_CONCEPT_STATUS; break; default: @@ -113,7 +110,7 @@ app.put('/submission-documents/:uuid', async function (req, res, next) { if (submission) { try { - if (submission.status == SENT_STATUS) { + if (submission.status == env.SENT_STATUS) { return res .status(422) .send({ title: `Submission ${submission.uri} already submitted` }); @@ -144,21 +141,21 @@ app.post('/submission-documents/:uuid/submit', async function (req, res, next) { if (submission) { try { - if (submission.status == SENT_STATUS) { + if (submission.status == env.SENT_STATUS) { return res .status(422) .send({ title: `Submission ${submission.uri} already submitted` }); } else { - await submission.updateStatus(SUBMITABLE_STATUS); + await submission.updateStatus(env.SUBMITABLE_STATUS); const newStatus = (await submission.process()).status; - if (newStatus == SENT_STATUS) { + if (newStatus == env.SENT_STATUS) { return res.status(204).send(); } else { return res.status(400).send({ title: 'Unable to submit form' }); } } } catch (error) { - await submission.updateStatus(CONCEPT_STATUS); + await submission.updateStatus(env.CONCEPT_STATUS); console.log( `Something went wrong while submitting submission with id ${uuid}` ); diff --git a/env.js b/env.js index ea92a1d..c03007b 100644 --- a/env.js +++ b/env.js @@ -2,3 +2,10 @@ export const TASK_SUCCESSFUL_CONCEPT_STATUS = 'http://lblod.data.gift/automatische-melding-statuses/successful-concept'; export const TASK_SUCCESSFUL_SENT_STATUS = 'http://lblod.data.gift/automatische-melding-statuses/successful-sent'; + +export const CONCEPT_STATUS = + 'http://lblod.data.gift/concepts/79a52da4-f491-4e2f-9374-89a13cde8ecd'; +export const SUBMITABLE_STATUS = + 'http://lblod.data.gift/concepts/f6330856-e261-430f-b949-8e510d20d0ff'; +export const SENT_STATUS = + 'http://lblod.data.gift/concepts/9bd8d86d-bb10-4456-a84e-91e9507c374c'; diff --git a/lib/file-helpers.js b/lib/file-helpers.js index 1b0128c..cc774a0 100644 --- a/lib/file-helpers.js +++ b/lib/file-helpers.js @@ -7,7 +7,7 @@ import { sparqlEscapeUri, uuid, } from 'mu'; -import * as env from '../env.js'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; /** * Returns the content of the given file @@ -31,7 +31,7 @@ export async function insertTtlFile(submissionDocument, content) { const filename = `${physicalId}.ttl`; const path = `/share/submissions/${filename}`; const physicalUri = path.replace('/share/', 'share://'); - const logicalUri = env.PREFIX_TABLE.asj.concat(logicalId); + const logicalUri = cts.PREFIX_TABLE.asj.concat(logicalId); const nowSparql = sparqlEscapeDateTime(new Date()); try { @@ -47,7 +47,7 @@ export async function insertTtlFile(submissionDocument, content) { //Sudo required because may be called both from automatic-submission or user await updateSudo(` - ${env.PREFIXES} + ${cts.SPARQL_PREFIXES} INSERT { GRAPH ?g { ${sparqlEscapeUri(physicalUri)} @@ -55,7 +55,7 @@ export async function insertTtlFile(submissionDocument, content) { mu:uuid ${sparqlEscapeString(physicalId)} ; nie:dataSource asj:${logicalId} ; nfo:fileName ${sparqlEscapeString(filename)} ; - dct:creator ${sparqlEscapeUri(env.CREATOR)} ; + dct:creator ${sparqlEscapeUri(cts.SERVICES.validateSubmission)} ; dct:created ${nowSparql} ; dct:modified ${nowSparql} ; dct:format "text/turtle" ; @@ -66,7 +66,7 @@ export async function insertTtlFile(submissionDocument, content) { a nfo:FileDataObject; mu:uuid ${sparqlEscapeString(logicalId)} ; nfo:fileName ${sparqlEscapeString(filename)} ; - dct:creator ${sparqlEscapeUri(env.CREATOR)} ; + dct:creator ${sparqlEscapeUri(cts.SERVICES.validateSubmission)} ; dct:created ${nowSparql} ; dct:modified ${nowSparql} ; dct:format "text/turtle" ; @@ -93,7 +93,7 @@ export async function updateTtlFile( content ) { const response = await querySudo(` - ${env.PREFIXES} + ${cts.SPARQL_PREFIXES} SELECT ?physicalUri WHERE { GRAPH ?g { ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . diff --git a/lib/remote-data-object.js b/lib/remote-data-object.js index 3027163..94cbc4a 100644 --- a/lib/remote-data-object.js +++ b/lib/remote-data-object.js @@ -5,7 +5,7 @@ import { sparqlEscapeUri, } from 'mu'; import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; -import * as env from '../env.js'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; import { NIE } from '@lblod/submission-form-helpers'; const DOWNLOAD_STATUS_READY_TO_BE_CASHED = @@ -136,7 +136,7 @@ export class RemoteDataObject { this.address = address; this.uuid = uuid(); this.status = DOWNLOAD_STATUS_READY_TO_BE_CASHED; - this.creator = env.CREATOR; + this.creator = cts.SERVICES.validateSubmission; this.created = new Date(); this.modified = new Date(); } diff --git a/lib/submission-task.js b/lib/submission-task.js index 8780fda..dd73fde 100644 --- a/lib/submission-task.js +++ b/lib/submission-task.js @@ -1,6 +1,6 @@ import * as mu from 'mu'; import * as mas from '@lblod/mu-auth-sudo'; -import * as env from '../env.js'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; /** * Updates the state of the given task to the specified status with potential error message or resultcontainer file @@ -18,7 +18,7 @@ export async function updateTaskStatus( ) { const taskUriSparql = mu.sparqlEscapeUri(taskUri); const nowSparql = mu.sparqlEscapeDateTime(new Date().toISOString()); - const hasError = errorUri && status === env.TASK_FAILURE_STATUS; + const hasError = errorUri && status === cts.TASK_STATUSES.failed; let resultContainerTriples = ''; let resultContainerUuid = ''; @@ -38,14 +38,14 @@ export async function updateTaskStatus( //TODO This triple does not do anything? Where is inputContainer defined? //Searched through toezicht-flattened-form-data-generator, and could not find ext:additionalStatus anywhere, so this can be removed later. const extraStatusTriple = - status === env.TASK_SUCCESS_STATUS && extraStatus + status === cts.TASK_SUCCESS_STATUS.success && extraStatus ? '?inputContainer ext:additionalStatus ' + mu.sparqlEscapeUri(extraStatus) + ' .' : ''; const statusUpdateQuery = ` - ${env.PREFIXES} + ${cts.SPARQL_PREFIXES} DELETE { GRAPH ?g { ${taskUriSparql} diff --git a/lib/submission.js b/lib/submission.js index e3f1245..74182e6 100644 --- a/lib/submission.js +++ b/lib/submission.js @@ -10,13 +10,7 @@ import { } from './submission-form'; import { RemoteDataObject } from './remote-data-object'; import * as env from '../env.js'; - -export const CONCEPT_STATUS = - 'http://lblod.data.gift/concepts/79a52da4-f491-4e2f-9374-89a13cde8ecd'; -export const SUBMITABLE_STATUS = - 'http://lblod.data.gift/concepts/f6330856-e261-430f-b949-8e510d20d0ff'; -export const SENT_STATUS = - 'http://lblod.data.gift/concepts/9bd8d86d-bb10-4456-a84e-91e9507c374c'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; export default class Submission { constructor({ uri, status, submittedResource }) { @@ -87,12 +81,12 @@ export default class Submission { let targetStatus = null; - if (currentStatus === SUBMITABLE_STATUS) { + if (currentStatus === env.SUBMITABLE_STATUS) { if (!isValid) { console.log( `Resetting status of submission ${this.uri} to concept since it's invalid` ); - targetStatus = CONCEPT_STATUS; + targetStatus = env.CONCEPT_STATUS; } else { console.log( `Updating status of submission ${this.uri} to sent state since it's valid` @@ -100,7 +94,7 @@ export default class Submission { await this.submit(formBuilder.form.value, triples); // NOTE find and save/update remote-data-objects await RemoteDataObject.process(triples); - targetStatus = SENT_STATUS; + targetStatus = env.SENT_STATUS; } } @@ -163,7 +157,7 @@ export default class Submission { export async function getSubmissionByTask(taskUri) { const response = await query(` - ${env.PREFIXES} + ${cts.SPARQL_PREFIXES} SELECT ?submission ?submissionDocument ?status WHERE { GRAPH ?g { From 1d87821fc495e7e83476c3de3120c03f911f6046 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 18:28:00 +0200 Subject: [PATCH 05/14] Submission task status via tools --- app.js | 28 +++++++++------ lib/submission-task.js | 78 ------------------------------------------ 2 files changed, 18 insertions(+), 88 deletions(-) delete mode 100644 lib/submission-task.js diff --git a/app.js b/app.js index 4007225..536c6b6 100644 --- a/app.js +++ b/app.js @@ -1,6 +1,5 @@ import { app, errorHandler } from 'mu'; import bodyParser from 'body-parser'; -import { updateTaskStatus } from './lib/submission-task'; import { getSubmissionByTask, getSubmissionBySubmissionDocument, @@ -46,7 +45,11 @@ app.post('/delta', async function (req, res) { for (const task of actualTasks) { const taskUri = task.value; try { - await updateTaskStatus(taskUri, cts.TASK_STATUSES.busy); + await tsk.updateStatus( + task, + namedNode(cts.TASK_STATUSES.busy), + namedNode(cts.SERVICES.validateSubmission) + ); const submission = await getSubmissionByTask(taskUri); const { status, logicalFileUri } = await submission.process(); @@ -65,23 +68,28 @@ app.post('/delta', async function (req, res) { break; } - await updateTaskStatus( - taskUri, - cts.TASK_STATUSES.success, - undefined, //Potential errorURI - saveStatus, - logicalFileUri + await tsk.updateStatus( + task, + namedNode(cts.TASK_STATUSES.success), + namedNode(cts.SERVICES.validateSubmission), + { files: [namedNode(logicalFileUri)] } ); } catch (error) { const message = `Something went wrong while enriching for task ${taskUri}`; console.error(`${message}\n`, error.message); console.error(error); - const errorUri = await err.create( + const errorNode = await err.create( namedNode(cts.SERVICES.validateSubmission), message, error.message ); - await updateTaskStatus(taskUri, cts.TASK_STATUSES.failed, errorUri); + await tsk.updateStatus( + task, + namedNode(cts.TASK_STATUSES.failed), + namedNode(cts.SERVICES.validateSubmission), + undefined, + namedNode(errorNode) + ); } } } catch (error) { diff --git a/lib/submission-task.js b/lib/submission-task.js deleted file mode 100644 index dd73fde..0000000 --- a/lib/submission-task.js +++ /dev/null @@ -1,78 +0,0 @@ -import * as mu from 'mu'; -import * as mas from '@lblod/mu-auth-sudo'; -import * as cts from '../automatic-submission-flow-tools/constants.js'; - -/** - * Updates the state of the given task to the specified status with potential error message or resultcontainer file - * - * @param string taskUri URI of the task - * @param string status URI of the new status - * @param string or undefined URI of the error that needs to be attached - */ -export async function updateTaskStatus( - taskUri, - status, - errorUri, - extraStatus, - logicalFileUri -) { - const taskUriSparql = mu.sparqlEscapeUri(taskUri); - const nowSparql = mu.sparqlEscapeDateTime(new Date().toISOString()); - const hasError = errorUri && status === cts.TASK_STATUSES.failed; - - let resultContainerTriples = ''; - let resultContainerUuid = ''; - if (logicalFileUri) { - resultContainerUuid = mu.uuid(); - resultContainerTriples = ` - asj:${resultContainerUuid} - a nfo:DataContainer ; - mu:uuid ${mu.sparqlEscapeString(resultContainerUuid)} ; - task:hasFile ${mu.sparqlEscapeUri(logicalFileUri)} . - `; - } - const resultsContainerLink = resultContainerUuid - ? `task:resultsContainer asj:${resultContainerUuid} ;` - : ''; - - //TODO This triple does not do anything? Where is inputContainer defined? - //Searched through toezicht-flattened-form-data-generator, and could not find ext:additionalStatus anywhere, so this can be removed later. - const extraStatusTriple = - status === cts.TASK_SUCCESS_STATUS.success && extraStatus - ? '?inputContainer ext:additionalStatus ' + - mu.sparqlEscapeUri(extraStatus) + - ' .' - : ''; - - const statusUpdateQuery = ` - ${cts.SPARQL_PREFIXES} - DELETE { - GRAPH ?g { - ${taskUriSparql} - adms:status ?oldStatus ; - dct:modified ?oldModified . - } - } - INSERT { - GRAPH ?g { - ${taskUriSparql} - adms:status ${mu.sparqlEscapeUri(status)} ; - ${resultsContainerLink} - ${hasError ? `task:error ${mu.sparqlEscapeUri(errorUri)} ;` : ''} - dct:modified ${nowSparql} . - - ${extraStatusTriple} - - ${resultContainerTriples} - } - } - WHERE { - GRAPH ?g { - ${taskUriSparql} - adms:status ?oldStatus ; - dct:modified ?oldModified . - } - } - `; - await mas.updateSudo(statusUpdateQuery); -} From 3c734f583d9674c41479606347c3e4c042b8619f Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 18:28:20 +0200 Subject: [PATCH 06/14] File access via tools --- lib/file-helpers.js | 157 ----------------------------------------- lib/submission-form.js | 151 ++++++++++++++++++++------------------- 2 files changed, 75 insertions(+), 233 deletions(-) delete mode 100644 lib/file-helpers.js diff --git a/lib/file-helpers.js b/lib/file-helpers.js deleted file mode 100644 index cc774a0..0000000 --- a/lib/file-helpers.js +++ /dev/null @@ -1,157 +0,0 @@ -import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; -import fs from 'fs-extra'; -import { - sparqlEscapeDateTime, - sparqlEscapeInt, - sparqlEscapeString, - sparqlEscapeUri, - uuid, -} from 'mu'; -import * as cts from '../automatic-submission-flow-tools/constants.js'; - -/** - * Returns the content of the given file - * - * @param string file URI of the file to get the content for - */ -export async function getFileContent(file) { - console.log(`Getting contents of file ${file}`); - const path = file.replace('share://', '/share/'); - return await fs.readFile(path, 'utf8'); -} - -/** - * Write the given TTL content to a file and relates it to the given submitted document - * - * @param string ttl Turtle to write to the file - */ -export async function insertTtlFile(submissionDocument, content) { - const logicalId = uuid(); - const physicalId = uuid(); - const filename = `${physicalId}.ttl`; - const path = `/share/submissions/${filename}`; - const physicalUri = path.replace('/share/', 'share://'); - const logicalUri = cts.PREFIX_TABLE.asj.concat(logicalId); - const nowSparql = sparqlEscapeDateTime(new Date()); - - try { - await fs.writeFile(path, content, 'utf-8'); - } catch (e) { - console.log(`Failed to write TTL to file <${path}>.`); - throw e; - } - - try { - const stats = await fs.stat(path); - const fileSize = stats.size; - - //Sudo required because may be called both from automatic-submission or user - await updateSudo(` - ${cts.SPARQL_PREFIXES} - INSERT { - GRAPH ?g { - ${sparqlEscapeUri(physicalUri)} - a nfo:FileDataObject ; - mu:uuid ${sparqlEscapeString(physicalId)} ; - nie:dataSource asj:${logicalId} ; - nfo:fileName ${sparqlEscapeString(filename)} ; - dct:creator ${sparqlEscapeUri(cts.SERVICES.validateSubmission)} ; - dct:created ${nowSparql} ; - dct:modified ${nowSparql} ; - dct:format "text/turtle" ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} ; - dbpedia:fileExtension "ttl" . - - asj:${logicalId} - a nfo:FileDataObject; - mu:uuid ${sparqlEscapeString(logicalId)} ; - nfo:fileName ${sparqlEscapeString(filename)} ; - dct:creator ${sparqlEscapeUri(cts.SERVICES.validateSubmission)} ; - dct:created ${nowSparql} ; - dct:modified ${nowSparql} ; - dct:format "text/turtle" ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} ; - dbpedia:fileExtension "ttl" . - } - } - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . - } - }`); - } catch (e) { - console.log(`Failed to write TTL resource <${logicalUri}> to triplestore.`); - throw e; - } - - return logicalUri; -} - -export async function updateTtlFile( - submissionDocument, - logicalFileUri, - content -) { - const response = await querySudo(` - ${cts.SPARQL_PREFIXES} - SELECT ?physicalUri WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . - ?physicalUri nie:dataSource ${sparqlEscapeUri(logicalFileUri)} . - } - } - `); - const physicalUri = response.results.bindings[0].physicalUri.value; - const path = physicalUri.replace('share://', '/share/'); - const now = new Date(); - - try { - await fs.writeFile(path, content, 'utf-8'); - } catch (e) { - console.log(`Failed to write TTL to file <${path}>.`); - throw e; - } - - try { - const stats = await fs.stat(path); - const fileSize = stats.size; - - //Sudo required because may be called both from automatic-submission or user - await updateSudo(` - PREFIX nfo: - PREFIX dct: - PREFIX ext: - - DELETE { - GRAPH ?g { - ${sparqlEscapeUri(physicalUri)} - dct:modified ?modified ; - nfo:fileSize ?fileSize . - ${sparqlEscapeUri(logicalFileUri)} - dct:modified ?modified ; - nfo:fileSize ?fileSize . - } - } - INSERT { - GRAPH ?g { - ${sparqlEscapeUri(physicalUri)} - dct:modified ${sparqlEscapeDateTime(now)} ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} . - ${sparqlEscapeUri(logicalFileUri)} - dct:modified ${sparqlEscapeDateTime(now)} ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} . - } - } - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . - } - } - `); - } catch (e) { - console.log( - `Failed to update TTL resource <${logicalFileUri}> in triplestore.` - ); - throw e; - } -} diff --git a/lib/submission-form.js b/lib/submission-form.js index 8ecdda5..5fa6ec5 100644 --- a/lib/submission-form.js +++ b/lib/submission-form.js @@ -1,8 +1,12 @@ import { uuid, sparqlEscapeUri } from 'mu'; -import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; -import { getFileContent, insertTtlFile, updateTtlFile } from './file-helpers'; +import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; import ForkingStore from 'forking-store'; import { NamedNode } from 'rdflib'; +import * as fil from '../automatic-submission-flow-tools/asfFiles.js'; +import * as sjp from 'sparqljson-parse'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; +import * as N3 from 'n3'; +const { namedNode } = N3.DataFactory; const FORM_DATA_FILE_TYPE = 'http://data.lblod.gift/concepts/form-data-file-type'; @@ -19,7 +23,7 @@ const FORM_FILE_TYPE = 'http://data.lblod.gift/concepts/form-file-type'; * @return {string} TTL with form description for the given submission document */ export function getFormTtl(submissionDocument) { - return getPartNoLogical(submissionDocument, FORM_FILE_TYPE); + return getPartWithoutLogical(submissionDocument, FORM_FILE_TYPE); } /** @@ -90,23 +94,25 @@ export async function saveFormTriples(submissionDocument, triples) { * @return {string} TTL with harvested data for the given submission document */ async function getHarvestedData(submissionDocument) { - const result = await query(` + const response = await querySudo(` PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?physicalFile + SELECT DISTINCT ?logicalFile WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?logicalFile . - ?logicalFile dct:type . - ?physicalFile nie:dataSource ?logicalFile . + ${sparqlEscapeUri(submissionDocument)} + dct:source ?logicalFile . + ?logicalFile + dct:type . } - } + } LIMIT 1 `); - if (result.results.bindings.length) { - const file = result.results.bindings[0]['physicalFile'].value; - return await getFileContent(file); + const sparqlJsonParser = new sjp.SparqlJsonParser(); + const parsedResults = sparqlJsonParser.parseJsonResults(response); + + if (parsedResults.length) { + const file = parsedResults[0].logicalFile; + return fil.loadFromLogicalFile(file); } } @@ -140,65 +146,42 @@ function getRemovals(submissionDocument) { * @return {string} Content of the related file */ async function getPart(submissionDocument, fileType) { - const result = await query(` - PREFIX mu: - PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?physicalFile - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?logicalFile . - ?physicalFile nie:dataSource ?logicalFile . - } - - ?logicalFile dct:type ${sparqlEscapeUri(fileType)} . - } - `); - - if (result.results.bindings.length) { - const file = result.results.bindings[0]['physicalFile'].value; - return await getFileContent(file); - } else { - console.log( - `No file of type ${fileType} found for submission document ${submissionDocument}` - ); - return null; - } + const file = await getFileResource(submissionDocument, fileType); + if (file) return fil.loadFromLogicalFile(file); +} +async function getPartWithoutLogical(submissionDocument, fileType) { + const file = await getFileResource(submissionDocument, fileType); + if (file) return fil.loadFromPhysicalFile(file); } /** - * Get the content of a file of the given file type that is related to the given submission document - * This function does not do it properly according to the file service. There is no logical file accompanying the physical file, only the physical file. This is to maintain compatibility with other services in the flow for now. + * Get the file resource in the triplestore of the given file type that is related to the given submission document * * @param {string} submissionDocument URI of the submitted document to get the related file for * @param {string} fileType URI of the type of the related file - * @return {string} Content of the related file + * @return {namedNode|undefined} File full name (path, name and extention) */ -async function getPartNoLogical(submissionDocument, fileType) { - const result = await query(` - PREFIX mu: +async function getFileResource(submissionDocument, fileType) { + const response = await querySudo(` PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?file + SELECT DISTINCT ?logicalFile WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?file . + ${sparqlEscapeUri(submissionDocument)} dct:source ?logicalFile . } - - ?file dct:type ${sparqlEscapeUri(fileType)} . - } + ?logicalFile dct:type ${sparqlEscapeUri(fileType)} . + } LIMIT 1 `); - if (result.results.bindings.length) { - const file = result.results.bindings[0]['file'].value; - return await getFileContent(file); + const sparqlJsonParser = new sjp.SparqlJsonParser(); + const parsedResults = sparqlJsonParser.parseJsonResults(response); + + if (parsedResults.length) { + return parsedResults[0].logicalFile; } else { console.log( - `No file of type ${fileType} found for submission document ${submissionDocument}` + `Part of type ${fileType} for submission document ${submissionDocument} not found` ); - return null; } } @@ -242,43 +225,59 @@ function saveFormData(submissionDocument, content) { * @param {string} fileType URI of the type of the related file */ async function savePart(submissionDocument, content, fileType) { - const result = await query(` - PREFIX mu: + const response = await querySudo(` PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?file + SELECT ?logicalFile WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?file . + ${sparqlEscapeUri(submissionDocument)} + dct:source ?logicalFile . + ?logicalFile + dct:type ${sparqlEscapeUri(fileType)} . } - ?file dct:type ${sparqlEscapeUri(fileType)} . - } + } LIMIT 0 `); - if (!result.results.bindings.length) { - const logicalFileUri = await insertTtlFile(submissionDocument, content); - await update(` + const sparqlJsonParser = new sjp.SparqlJsonParser(); + const parsedResults = sparqlJsonParser.parseJsonResults(response); + + if (!parsedResults.length) { + const graphResponse = await querySudo(` + PREFIX ext: + SELECT ?g WHERE { + GRAPH ?g { + ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . + } + } LIMIT 1 + `); + const graphResults = sparqlJsonParser.parseJsonResults(graphResponse); + const graph = graphResults[0]?.g; + const { logicalFile } = await fil.createFromContent( + content, + namedNode(cts.SERVICES.enrichSubmission), + graph + ); + await updateSudo(` PREFIX dct: PREFIX ext: - INSERT { GRAPH ?g { ${sparqlEscapeUri(submissionDocument)} - dct:source ${sparqlEscapeUri(logicalFileUri)} . - ${sparqlEscapeUri(logicalFileUri)} + dct:source ${sparqlEscapeUri(logicalFile.value)} . + ${sparqlEscapeUri(logicalFile.value)} dct:type ${sparqlEscapeUri(fileType)} . } } WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . + ${sparqlEscapeUri(submissionDocument)} + a ext:SubmissionDocument . } } `); - return logicalFileUri; + return logicalFile.value; } else { - const file = result.results.bindings[0]['file'].value; - await updateTtlFile(submissionDocument, file, content); - return file; + const logicalFile = parsedResults[0].logicalFile; + await fil.updateContentForLogicalFile(logicalFile, content); + return logicalFile; } } From 1e4ad46a1bf826da22c57940626f57de01690e07 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 22:19:52 +0200 Subject: [PATCH 07/14] Moved vars to env.js file --- lib/form-builder.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/form-builder.js b/lib/form-builder.js index 963c336..919c43f 100644 --- a/lib/form-builder.js +++ b/lib/form-builder.js @@ -4,31 +4,28 @@ import { importTriplesForForm, validateForm, } from '@lblod/submission-form-helpers'; - -const FORM_GRAPH = 'http://data.lblod.info/graphs/semantic-forms'; -const META_GRAPH = 'http://data.lblod.info/graphs/meta'; -const SOURCE_GRAPH = 'http://data.lblod.info/graphs/submission'; +import * as env from '../env.js'; export default class FormBuilder { - constructor({ submittedResource, formTtl, sourceTtl, metaTtl }) { + constructor(submittedResource, formTtl, sourceTtl, metaTtl) { this.store = rdflibGraph(); - rdflibParse(formTtl, this.store, FORM_GRAPH, 'text/turtle'); - rdflibParse(metaTtl, this.store, META_GRAPH, 'text/turtle'); - rdflibParse(sourceTtl, this.store, SOURCE_GRAPH, 'text/turtle'); + rdflibParse(formTtl, this.store, env.FORM_GRAPH, 'text/turtle'); + rdflibParse(metaTtl, this.store, env.META_GRAPH, 'text/turtle'); + rdflibParse(sourceTtl, this.store, env.SOURCE_GRAPH, 'text/turtle'); this._submittedResource = submittedResource; } get formGraph() { - return new NamedNode(FORM_GRAPH); + return new NamedNode(env.FORM_GRAPH); } get metaGraph() { - return new NamedNode(META_GRAPH); + return new NamedNode(env.META_GRAPH); } get sourceGraph() { - return new NamedNode(SOURCE_GRAPH); + return new NamedNode(env.SOURCE_GRAPH); } get submittedResource() { From f766d6e310c2ba9e327819764893f409ec34239b Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 22:20:24 +0200 Subject: [PATCH 08/14] Removed awkward object notation from constructor --- lib/remote-data-object.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/remote-data-object.js b/lib/remote-data-object.js index 94cbc4a..f4472fd 100644 --- a/lib/remote-data-object.js +++ b/lib/remote-data-object.js @@ -105,7 +105,9 @@ export class RemoteDataObject { */ static async saveCollection(remotes) { const remotesTriples = remotes - .map((remote) => new RemoteDataObject(remote).toSPARQL()) + .map((remote) => + new RemoteDataObject(remote.uri, remote.address).toSPARQL() + ) .join('\n '); const q = ` PREFIX nfo: @@ -131,7 +133,7 @@ export class RemoteDataObject { } } - constructor({ uri, address }) { + constructor(uri, address) { this.uri = uri; this.address = address; this.uuid = uuid(); From 1f927100f2e6c480d2bd1fdf1dc676b3249ae320 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 22:21:11 +0200 Subject: [PATCH 09/14] Removed more awkward object notations --- app.js | 2 +- env.js | 4 ++++ lib/submission.js | 12 ++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index 536c6b6..28a29fe 100644 --- a/app.js +++ b/app.js @@ -124,7 +124,7 @@ app.put('/submission-documents/:uuid', async function (req, res, next) { .send({ title: `Submission ${submission.uri} already submitted` }); } else { const { additions, removals } = req.body; - await submission.update({ additions, removals }); + await submission.update(additions, removals); return res.status(204).send(); } } catch (e) { diff --git a/env.js b/env.js index c03007b..3783bdf 100644 --- a/env.js +++ b/env.js @@ -9,3 +9,7 @@ export const SUBMITABLE_STATUS = 'http://lblod.data.gift/concepts/f6330856-e261-430f-b949-8e510d20d0ff'; export const SENT_STATUS = 'http://lblod.data.gift/concepts/9bd8d86d-bb10-4456-a84e-91e9507c374c'; + +export const FORM_GRAPH = 'http://data.lblod.info/graphs/semantic-forms'; +export const META_GRAPH = 'http://data.lblod.info/graphs/meta'; +export const SOURCE_GRAPH = 'http://data.lblod.info/graphs/submission'; diff --git a/lib/submission.js b/lib/submission.js index 74182e6..87ea72e 100644 --- a/lib/submission.js +++ b/lib/submission.js @@ -13,7 +13,7 @@ import * as env from '../env.js'; import * as cts from '../automatic-submission-flow-tools/constants.js'; export default class Submission { - constructor({ uri, status, submittedResource }) { + constructor(uri, status, submittedResource) { this.uri = uri; this.status = status; this.submittedResource = submittedResource; // submission document @@ -43,7 +43,7 @@ export default class Submission { /** * Updates the additions and removals and processes the form */ - async update({ additions, removals }) { + async update(additions, removals) { await updateDocument(this.submittedResource, { additions, removals }); await this.process(); } @@ -56,12 +56,12 @@ export default class Submission { const formTtl = await getFormTtl(this.submittedResource); const metaTtl = await getMetaTtl(this.submittedResource); const sourceTtl = await getSourceTtl(this.submittedResource); - const formBuilder = new FormBuilder({ - submittedResource: this.submittedResource, + const formBuilder = new FormBuilder( + this.submittedResource, formTtl, sourceTtl, - metaTtl, - }); + metaTtl + ); const triples = formBuilder.build().data(); if (!triples.length) { From 9d4cadbb3f9462f1b11138087155733d00c76cf6 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Wed, 5 Oct 2022 22:22:27 +0200 Subject: [PATCH 10/14] Rewrite of some function with tools --- automatic-submission-flow-tools | 2 +- lib/submission-form.js | 6 +- lib/submission.js | 101 ++++++++------------------------ 3 files changed, 30 insertions(+), 79 deletions(-) diff --git a/automatic-submission-flow-tools b/automatic-submission-flow-tools index 2718b2f..bcf0fd7 160000 --- a/automatic-submission-flow-tools +++ b/automatic-submission-flow-tools @@ -1 +1 @@ -Subproject commit 2718b2ffe336f6f0140bc088857cd1b199faa3a2 +Subproject commit bcf0fd785fa48c54ee57ce62f1135cb0cee6ce3d diff --git a/lib/submission-form.js b/lib/submission-form.js index 5fa6ec5..67b6052 100644 --- a/lib/submission-form.js +++ b/lib/submission-form.js @@ -82,9 +82,9 @@ export async function saveFormTriples(submissionDocument, triples) { return file; } -/* - * Private - */ +//////////////////////////////////////////////////////////////////////////////// +// Shared with enrich-submission-service +//////////////////////////////////////////////////////////////////////////////// /** * Get harvested data of a submission document in TTL format. diff --git a/lib/submission.js b/lib/submission.js index 87ea72e..0241459 100644 --- a/lib/submission.js +++ b/lib/submission.js @@ -1,5 +1,5 @@ -import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu'; -import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; +import { sparqlEscapeUri, sparqlEscapeDateTime } from 'mu'; +import { updateSudo as update } from '@lblod/mu-auth-sudo'; import FormBuilder from './form-builder'; import { getFormTtl, @@ -11,6 +11,9 @@ import { import { RemoteDataObject } from './remote-data-object'; import * as env from '../env.js'; import * as cts from '../automatic-submission-flow-tools/constants.js'; +import * as smt from '../automatic-submission-flow-tools/asfSubmissions.js'; +import * as N3 from 'n3'; +const { namedNode, literal } = N3.DataFactory; export default class Submission { constructor(uri, status, submittedResource) { @@ -21,8 +24,7 @@ export default class Submission { async updateStatus(status) { await update(` - PREFIX adms: - + ${cts.SPARQL_PREFIXES} DELETE { GRAPH ?g { ${sparqlEscapeUri(this.uri)} adms:status ?status . @@ -75,7 +77,8 @@ export default class Submission { `Form for submitted resource ${this.submittedResource} is valid: ${isValid}` ); - const currentStatus = await getSubmissionStatus(this.uri); + const submissionInfo = await smt.getSubmissionInfo(namedNode(this.uri)); + const currentStatus = submissionInfo?.status?.value; if (!currentStatus) throw new Error(`Submission <${this.uri}> doesn't have a status`); @@ -156,77 +159,25 @@ export default class Submission { } export async function getSubmissionByTask(taskUri) { - const response = await query(` - ${cts.SPARQL_PREFIXES} - SELECT ?submission ?submissionDocument ?status - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(taskUri)} - a task:Task ; - dct:isPartOf ?job . - ?job prov:generatedBy ?submission . - ?submission - dct:subject ?submissionDocument ; - adms:status ?status . - } - } LIMIT 1 - `); - - const bindings = response?.results?.bindings; - if (bindings && bindings.length > 0) { - const binding = bindings[0]; - return new Submission({ - uri: binding.submission.value, - status: binding.status.value, - submittedResource: binding.submissionDocument.value, - }); - } + const submissionInfo = await smt.getSubmissionInfoFromTask( + namedNode(taskUri) + ); + if (submissionInfo) + return new Submission( + submissionInfo.submission.value, + submissionInfo.status.value, + submissionInfo.submittedDocument.value + ); } export async function getSubmissionBySubmissionDocument(uuid) { - const result = await query(` - PREFIX nie: - PREFIX prov: - PREFIX melding: - PREFIX adms: - PREFIX dct: - PREFIX mu: - - SELECT ?submission ?submissionDocument ?status - WHERE { - GRAPH ?g { - ?submissionDocument mu:uuid ${sparqlEscapeString(uuid)} . - ?submission - dct:subject ?submissionDocument ; - adms:status ?status . - } - } LIMIT 1 - `); - - if (result.results.bindings.length) { - const binding = result.results.bindings[0]; - return new Submission({ - uri: binding['submission'].value, - status: binding['status'].value, - submittedResource: binding['submissionDocument'].value, - }); - } else { - return null; - } -} - -export async function getSubmissionStatus(submissionUri) { - const result = await query(` - PREFIX adms: - - SELECT ?status - WHERE { ${sparqlEscapeUri(submissionUri)} adms:status ?status . } - LIMIT 1 -`); - - if (result.results.bindings.length) { - return result.results.bindings[0]['status'].value; - } else { - return null; - } + const submissionInfo = await smt.getSubmissionInfoFromSubmissionDocumentId( + literal(uuid) + ); + if (submissionInfo) + return new Submission( + submissionInfo.submission.value, + submissionInfo.status.value, + submissionInfo.submittedDocument.value + ); } From 6d3fb7626c61ffc7442c6fdde123ccbbbbe8214f Mon Sep 17 00:00:00 2001 From: benjay10 Date: Thu, 6 Oct 2022 20:17:41 +0200 Subject: [PATCH 11/14] Bugfix on writing errors state to task --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index 28a29fe..ac300a0 100644 --- a/app.js +++ b/app.js @@ -88,7 +88,7 @@ app.post('/delta', async function (req, res) { namedNode(cts.TASK_STATUSES.failed), namedNode(cts.SERVICES.validateSubmission), undefined, - namedNode(errorNode) + errorNode ); } } From bf6d15d9f70b701340bc5e6a2aaee4b48b4dff50 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Thu, 6 Oct 2022 20:18:18 +0200 Subject: [PATCH 12/14] Rewrite without ForkingStore, removed more packages When removing unused packages, the ForkingStore jumped out as only being used sparingly and something that does not provide much documentation. So this was removed in favour of some basic transformations with N3 stores. I'm sorry, but if there is no documentation on a library, I'm not using it, and I will convert code to using libraries that are properly documented and well tested. I don't want to debug something like that otherwise. --- automatic-submission-flow-tools | 2 +- lib/submission-form.js | 30 +++++++++++++----------------- package.json | 5 ----- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/automatic-submission-flow-tools b/automatic-submission-flow-tools index bcf0fd7..c5f5bdc 160000 --- a/automatic-submission-flow-tools +++ b/automatic-submission-flow-tools @@ -1 +1 @@ -Subproject commit bcf0fd785fa48c54ee57ce62f1135cb0cee6ce3d +Subproject commit c5f5bdc72b2c5fe0f480fb3569d9a50f0fbe6930 diff --git a/lib/submission-form.js b/lib/submission-form.js index 67b6052..09511f0 100644 --- a/lib/submission-form.js +++ b/lib/submission-form.js @@ -1,10 +1,9 @@ -import { uuid, sparqlEscapeUri } from 'mu'; +import { sparqlEscapeUri } from 'mu'; import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; -import ForkingStore from 'forking-store'; -import { NamedNode } from 'rdflib'; import * as fil from '../automatic-submission-flow-tools/asfFiles.js'; import * as sjp from 'sparqljson-parse'; import * as cts from '../automatic-submission-flow-tools/constants.js'; +import * as uti from '../automatic-submission-flow-tools/utils.js'; import * as N3 from 'n3'; const { namedNode } = N3.DataFactory; @@ -34,21 +33,18 @@ export function getFormTtl(submissionDocument) { * @return {string} TTL with source data for the given submission document */ export async function getSourceTtl(submissionDocument) { - const source = await getHarvestedData(submissionDocument); - const additions = await getAdditions(submissionDocument); - const removals = await getRemovals(submissionDocument); + const sourceContent = await getHarvestedData(submissionDocument); + const additionsContent = await getAdditions(submissionDocument); + const removalsContent = await getRemovals(submissionDocument); - // merge source, additions and removals - const forkingStore = new ForkingStore(); - const graph = new NamedNode(`http://merged-form/graph/${uuid()}`); - forkingStore.loadDataWithAddAndDelGraph( - source || '', - graph, - additions || '', - removals || '', - 'text/turtle' - ); - return forkingStore.serializeDataMergedGraph(graph, 'text/turtle'); + const base = await uti.ttlToStore(sourceContent); + const additions = await uti.ttlToStore(additionsContent); + const removals = await uti.ttlToStore(removalsContent); + + base.removeQuads([...removals]); + base.addQuads([...additions]); + + return uti.storeToTtl(base); } /** diff --git a/package.json b/package.json index 8ed4b15..403deb2 100644 --- a/package.json +++ b/package.json @@ -17,11 +17,6 @@ "rdf-string-ttl": "github:benjay10/rdf-string-ttl.js", "sparqljson-parse": "^2.1.1", "uuid": "^9.0.0", - - "forking-store": "^1.0.0", - "fs-extra": "^8.1.0", - "lodash.flatten": "^4.4.0", - "moment": "^2.24.0", "rdflib": "1.1.0" }, "devDependencies": { From ac78906122a6ce6ad3f4d29acbe34107320daf83 Mon Sep 17 00:00:00 2001 From: benjay10 Date: Thu, 6 Oct 2022 21:57:36 +0200 Subject: [PATCH 13/14] Removed more unused code --- app.js | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/app.js b/app.js index ac300a0..dae64fa 100644 --- a/app.js +++ b/app.js @@ -8,7 +8,6 @@ import * as env from './env.js'; import * as cts from './automatic-submission-flow-tools/constants.js'; import * as tsk from './automatic-submission-flow-tools/asfTasks.js'; import * as del from './automatic-submission-flow-tools/deltas.js'; -import * as smt from './automatic-submission-flow-tools/asfSubmissions.js'; import * as err from './automatic-submission-flow-tools/errors.js'; import * as N3 from 'n3'; const { namedNode } = N3.DataFactory; @@ -52,21 +51,7 @@ app.post('/delta', async function (req, res) { ); const submission = await getSubmissionByTask(taskUri); - const { status, logicalFileUri } = await submission.process(); - const resultingStatus = status; - - let saveStatus; - switch (resultingStatus) { - case env.SENT_STATUS: - saveStatus = env.TASK_SUCCESSFUL_SENT_STATUS; - break; - case env.CONCEPT_STATUS: - saveStatus = env.TASK_SUCCESSFUL_CONCEPT_STATUS; - break; - default: - saveStatus = resultingStatus; - break; - } + const { logicalFileUri } = await submission.process(); await tsk.updateStatus( task, From 4549695f0b994cf40a4da0d3258e2b51597d90be Mon Sep 17 00:00:00 2001 From: benjay10 Date: Sat, 8 Oct 2022 04:34:17 +0200 Subject: [PATCH 14/14] Upgraded Dockerfile JavaScript template --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5033ade..a64a004 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,2 @@ -FROM semtech/mu-javascript-template:1.5.0-beta.4 +FROM semtech/mu-javascript-template:feature-node-16-support LABEL maintainer=info@redpencil - -# see https://github.com/mu-semtech/mu-javascript-template for more info