diff --git a/.babelrc.json b/.babelrc.json new file mode 100644 index 0000000..dcb79ad --- /dev/null +++ b/.babelrc.json @@ -0,0 +1,9 @@ +{ + "overrides": [ + { "ignore": "./automatic-submission-flow-tools/node_modules" } + ], + "babelrcRoots": [ + ".", + "./packages/*" + ] +} diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fd994a2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "automatic-submission-flow-tools"] + path = automatic-submission-flow-tools + url = git@github.com:lblod/automatic-submission-flow-tools.git diff --git a/Dockerfile b/Dockerfile index 5033ade..a64a004 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,2 @@ -FROM semtech/mu-javascript-template:1.5.0-beta.4 +FROM semtech/mu-javascript-template:feature-node-16-support LABEL maintainer=info@redpencil - -# see https://github.com/mu-semtech/mu-javascript-template for more info diff --git a/app.js b/app.js index 95c3846..dae64fa 100644 --- a/app.js +++ b/app.js @@ -1,15 +1,16 @@ import { app, errorHandler } from 'mu'; import bodyParser from 'body-parser'; -import { updateTaskStatus } from './lib/submission-task'; import { getSubmissionByTask, getSubmissionBySubmissionDocument, - SUBMITABLE_STATUS, - SENT_STATUS, - CONCEPT_STATUS, } from './lib/submission'; import * as env from './env.js'; -import { saveError } from './lib/utils.js'; +import * as cts from './automatic-submission-flow-tools/constants.js'; +import * as tsk from './automatic-submission-flow-tools/asfTasks.js'; +import * as del from './automatic-submission-flow-tools/deltas.js'; +import * as err from './automatic-submission-flow-tools/errors.js'; +import * as N3 from 'n3'; +const { namedNode } = N3.DataFactory; app.use(errorHandler); app.use( @@ -34,48 +35,46 @@ app.post('/delta', async function (req, res) { try { //Don't trust the delta-notifier, filter as best as possible. We just need the task that was created to get started. - const actualTaskUris = req.body - .map((changeset) => changeset.inserts) - .filter((inserts) => inserts.length > 0) - .flat() - .filter((insert) => insert.predicate.value === env.OPERATION_PREDICATE) - .filter((insert) => insert.object.value === env.VALIDATE_OPERATION) - .map((insert) => insert.subject.value); + const actualTasks = del.getSubjects( + req.body, + namedNode(cts.PREDICATE_TABLE.task_operation), + namedNode(cts.OPERATIONS.validate) + ); - for (const taskUri of actualTaskUris) { + for (const task of actualTasks) { + const taskUri = task.value; try { - await updateTaskStatus(taskUri, env.TASK_ONGOING_STATUS); + await tsk.updateStatus( + task, + namedNode(cts.TASK_STATUSES.busy), + namedNode(cts.SERVICES.validateSubmission) + ); const submission = await getSubmissionByTask(taskUri); - const { status, logicalFileUri } = await submission.process(); - const resultingStatus = status; - - let saveStatus; - switch (resultingStatus) { - case SENT_STATUS: - saveStatus = env.TASK_SUCCESSFUL_SENT_STATUS; - break; - case CONCEPT_STATUS: - saveStatus = env.TASK_SUCCESSFUL_CONCEPT_STATUS; - break; - default: - saveStatus = resultingStatus; - break; - } + const { logicalFileUri } = await submission.process(); - await updateTaskStatus( - taskUri, - env.TASK_SUCCESS_STATUS, - undefined, //Potential errorURI - saveStatus, - logicalFileUri + await tsk.updateStatus( + task, + namedNode(cts.TASK_STATUSES.success), + namedNode(cts.SERVICES.validateSubmission), + { files: [namedNode(logicalFileUri)] } ); } catch (error) { const message = `Something went wrong while enriching for task ${taskUri}`; console.error(`${message}\n`, error.message); console.error(error); - const errorUri = await saveError({ message, detail: error.message }); - await updateTaskStatus(taskUri, env.TASK_FAILURE_STATUS, errorUri); + const errorNode = await err.create( + namedNode(cts.SERVICES.validateSubmission), + message, + error.message + ); + await tsk.updateStatus( + task, + namedNode(cts.TASK_STATUSES.failed), + namedNode(cts.SERVICES.validateSubmission), + undefined, + errorNode + ); } } } catch (error) { @@ -83,7 +82,11 @@ app.post('/delta', async function (req, res) { 'The task for enriching a submission could not even be started or finished due to an unexpected problem.'; console.error(`${message}\n`, error.message); console.error(error); - await saveError({ message, detail: error.message }); + await err.create( + namedNode(cts.SERVICES.validateSubmission), + message, + error.message + ); } }); @@ -100,13 +103,13 @@ app.put('/submission-documents/:uuid', async function (req, res, next) { if (submission) { try { - if (submission.status == SENT_STATUS) { + if (submission.status == env.SENT_STATUS) { return res .status(422) .send({ title: `Submission ${submission.uri} already submitted` }); } else { const { additions, removals } = req.body; - await submission.update({ additions, removals }); + await submission.update(additions, removals); return res.status(204).send(); } } catch (e) { @@ -131,21 +134,21 @@ app.post('/submission-documents/:uuid/submit', async function (req, res, next) { if (submission) { try { - if (submission.status == SENT_STATUS) { + if (submission.status == env.SENT_STATUS) { return res .status(422) .send({ title: `Submission ${submission.uri} already submitted` }); } else { - await submission.updateStatus(SUBMITABLE_STATUS); + await submission.updateStatus(env.SUBMITABLE_STATUS); const newStatus = (await submission.process()).status; - if (newStatus == SENT_STATUS) { + if (newStatus == env.SENT_STATUS) { return res.status(204).send(); } else { return res.status(400).send({ title: 'Unable to submit form' }); } } } catch (error) { - await submission.updateStatus(CONCEPT_STATUS); + await submission.updateStatus(env.CONCEPT_STATUS); console.log( `Something went wrong while submitting submission with id ${uuid}` ); diff --git a/automatic-submission-flow-tools b/automatic-submission-flow-tools new file mode 160000 index 0000000..c5f5bdc --- /dev/null +++ b/automatic-submission-flow-tools @@ -0,0 +1 @@ +Subproject commit c5f5bdc72b2c5fe0f480fb3569d9a50f0fbe6930 diff --git a/env.js b/env.js index b42cb0b..3783bdf 100644 --- a/env.js +++ b/env.js @@ -1,54 +1,15 @@ -export const CREATOR = - 'http://lblod.data.gift/services/validate-submission-service'; - -export const TASK_ONGOING_STATUS = - 'http://redpencil.data.gift/id/concept/JobStatus/busy'; -export const TASK_SUCCESS_STATUS = - 'http://redpencil.data.gift/id/concept/JobStatus/success'; -export const TASK_FAILURE_STATUS = - 'http://redpencil.data.gift/id/concept/JobStatus/failed'; - export const TASK_SUCCESSFUL_CONCEPT_STATUS = 'http://lblod.data.gift/automatische-melding-statuses/successful-concept'; export const TASK_SUCCESSFUL_SENT_STATUS = 'http://lblod.data.gift/automatische-melding-statuses/successful-sent'; -export const OPERATION_PREDICATE = - 'http://redpencil.data.gift/vocabularies/tasks/operation'; -export const VALIDATE_OPERATION = - 'http://lblod.data.gift/id/jobs/concept/TaskOperation/validate'; - -export const PREFIX_TABLE = { - meb: 'http://rdf.myexperiment.org/ontologies/base/', - xsd: 'http://www.w3.org/2001/XMLSchema#', - pav: 'http://purl.org/pav/', - dct: 'http://purl.org/dc/terms/', - melding: 'http://lblod.data.gift/vocabularies/automatische-melding/', - lblodBesluit: 'http://lblod.data.gift/vocabularies/besluit/', - adms: 'http://www.w3.org/ns/adms#', - muAccount: 'http://mu.semte.ch/vocabularies/account/', - eli: 'http://data.europa.eu/eli/ontology#', - org: 'http://www.w3.org/ns/org#', - elod: 'http://linkedeconomy.org/ontology#', - nie: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#', - prov: 'http://www.w3.org/ns/prov#', - mu: 'http://mu.semte.ch/vocabularies/core/', - foaf: 'http://xmlns.com/foaf/0.1/', - nfo: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#', - ext: 'http://mu.semte.ch/vocabularies/ext/', - http: 'http://www.w3.org/2011/http#', - rpioHttp: 'http://redpencil.data.gift/vocabularies/http/', - dgftSec: 'http://lblod.data.gift/vocabularies/security/', - dgftOauth: 'http://kanselarij.vo.data.gift/vocabularies/oauth-2.0-session/', - wotSec: 'https://www.w3.org/2019/wot/security#', - task: 'http://redpencil.data.gift/vocabularies/tasks/', - asj: 'http://data.lblod.info/id/automatic-submission-job/', - dbpedia: 'http://dbpedia.org/ontology/', -}; +export const CONCEPT_STATUS = + 'http://lblod.data.gift/concepts/79a52da4-f491-4e2f-9374-89a13cde8ecd'; +export const SUBMITABLE_STATUS = + 'http://lblod.data.gift/concepts/f6330856-e261-430f-b949-8e510d20d0ff'; +export const SENT_STATUS = + 'http://lblod.data.gift/concepts/9bd8d86d-bb10-4456-a84e-91e9507c374c'; -export const PREFIXES = (() => { - const all = []; - for (const key in PREFIX_TABLE) - all.push(`PREFIX ${key}: <${PREFIX_TABLE[key]}>`); - return all.join('\n'); -})(); +export const FORM_GRAPH = 'http://data.lblod.info/graphs/semantic-forms'; +export const META_GRAPH = 'http://data.lblod.info/graphs/meta'; +export const SOURCE_GRAPH = 'http://data.lblod.info/graphs/submission'; diff --git a/lib/file-helpers.js b/lib/file-helpers.js deleted file mode 100644 index 1b0128c..0000000 --- a/lib/file-helpers.js +++ /dev/null @@ -1,157 +0,0 @@ -import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; -import fs from 'fs-extra'; -import { - sparqlEscapeDateTime, - sparqlEscapeInt, - sparqlEscapeString, - sparqlEscapeUri, - uuid, -} from 'mu'; -import * as env from '../env.js'; - -/** - * Returns the content of the given file - * - * @param string file URI of the file to get the content for - */ -export async function getFileContent(file) { - console.log(`Getting contents of file ${file}`); - const path = file.replace('share://', '/share/'); - return await fs.readFile(path, 'utf8'); -} - -/** - * Write the given TTL content to a file and relates it to the given submitted document - * - * @param string ttl Turtle to write to the file - */ -export async function insertTtlFile(submissionDocument, content) { - const logicalId = uuid(); - const physicalId = uuid(); - const filename = `${physicalId}.ttl`; - const path = `/share/submissions/${filename}`; - const physicalUri = path.replace('/share/', 'share://'); - const logicalUri = env.PREFIX_TABLE.asj.concat(logicalId); - const nowSparql = sparqlEscapeDateTime(new Date()); - - try { - await fs.writeFile(path, content, 'utf-8'); - } catch (e) { - console.log(`Failed to write TTL to file <${path}>.`); - throw e; - } - - try { - const stats = await fs.stat(path); - const fileSize = stats.size; - - //Sudo required because may be called both from automatic-submission or user - await updateSudo(` - ${env.PREFIXES} - INSERT { - GRAPH ?g { - ${sparqlEscapeUri(physicalUri)} - a nfo:FileDataObject ; - mu:uuid ${sparqlEscapeString(physicalId)} ; - nie:dataSource asj:${logicalId} ; - nfo:fileName ${sparqlEscapeString(filename)} ; - dct:creator ${sparqlEscapeUri(env.CREATOR)} ; - dct:created ${nowSparql} ; - dct:modified ${nowSparql} ; - dct:format "text/turtle" ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} ; - dbpedia:fileExtension "ttl" . - - asj:${logicalId} - a nfo:FileDataObject; - mu:uuid ${sparqlEscapeString(logicalId)} ; - nfo:fileName ${sparqlEscapeString(filename)} ; - dct:creator ${sparqlEscapeUri(env.CREATOR)} ; - dct:created ${nowSparql} ; - dct:modified ${nowSparql} ; - dct:format "text/turtle" ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} ; - dbpedia:fileExtension "ttl" . - } - } - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . - } - }`); - } catch (e) { - console.log(`Failed to write TTL resource <${logicalUri}> to triplestore.`); - throw e; - } - - return logicalUri; -} - -export async function updateTtlFile( - submissionDocument, - logicalFileUri, - content -) { - const response = await querySudo(` - ${env.PREFIXES} - SELECT ?physicalUri WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . - ?physicalUri nie:dataSource ${sparqlEscapeUri(logicalFileUri)} . - } - } - `); - const physicalUri = response.results.bindings[0].physicalUri.value; - const path = physicalUri.replace('share://', '/share/'); - const now = new Date(); - - try { - await fs.writeFile(path, content, 'utf-8'); - } catch (e) { - console.log(`Failed to write TTL to file <${path}>.`); - throw e; - } - - try { - const stats = await fs.stat(path); - const fileSize = stats.size; - - //Sudo required because may be called both from automatic-submission or user - await updateSudo(` - PREFIX nfo: - PREFIX dct: - PREFIX ext: - - DELETE { - GRAPH ?g { - ${sparqlEscapeUri(physicalUri)} - dct:modified ?modified ; - nfo:fileSize ?fileSize . - ${sparqlEscapeUri(logicalFileUri)} - dct:modified ?modified ; - nfo:fileSize ?fileSize . - } - } - INSERT { - GRAPH ?g { - ${sparqlEscapeUri(physicalUri)} - dct:modified ${sparqlEscapeDateTime(now)} ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} . - ${sparqlEscapeUri(logicalFileUri)} - dct:modified ${sparqlEscapeDateTime(now)} ; - nfo:fileSize ${sparqlEscapeInt(fileSize)} . - } - } - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . - } - } - `); - } catch (e) { - console.log( - `Failed to update TTL resource <${logicalFileUri}> in triplestore.` - ); - throw e; - } -} diff --git a/lib/form-builder.js b/lib/form-builder.js index 963c336..919c43f 100644 --- a/lib/form-builder.js +++ b/lib/form-builder.js @@ -4,31 +4,28 @@ import { importTriplesForForm, validateForm, } from '@lblod/submission-form-helpers'; - -const FORM_GRAPH = 'http://data.lblod.info/graphs/semantic-forms'; -const META_GRAPH = 'http://data.lblod.info/graphs/meta'; -const SOURCE_GRAPH = 'http://data.lblod.info/graphs/submission'; +import * as env from '../env.js'; export default class FormBuilder { - constructor({ submittedResource, formTtl, sourceTtl, metaTtl }) { + constructor(submittedResource, formTtl, sourceTtl, metaTtl) { this.store = rdflibGraph(); - rdflibParse(formTtl, this.store, FORM_GRAPH, 'text/turtle'); - rdflibParse(metaTtl, this.store, META_GRAPH, 'text/turtle'); - rdflibParse(sourceTtl, this.store, SOURCE_GRAPH, 'text/turtle'); + rdflibParse(formTtl, this.store, env.FORM_GRAPH, 'text/turtle'); + rdflibParse(metaTtl, this.store, env.META_GRAPH, 'text/turtle'); + rdflibParse(sourceTtl, this.store, env.SOURCE_GRAPH, 'text/turtle'); this._submittedResource = submittedResource; } get formGraph() { - return new NamedNode(FORM_GRAPH); + return new NamedNode(env.FORM_GRAPH); } get metaGraph() { - return new NamedNode(META_GRAPH); + return new NamedNode(env.META_GRAPH); } get sourceGraph() { - return new NamedNode(SOURCE_GRAPH); + return new NamedNode(env.SOURCE_GRAPH); } get submittedResource() { diff --git a/lib/remote-data-object.js b/lib/remote-data-object.js index f8ddff6..f4472fd 100644 --- a/lib/remote-data-object.js +++ b/lib/remote-data-object.js @@ -5,7 +5,7 @@ import { sparqlEscapeUri, } from 'mu'; import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; -import * as env from '../env.js'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; import { NIE } from '@lblod/submission-form-helpers'; const DOWNLOAD_STATUS_READY_TO_BE_CASHED = @@ -63,7 +63,7 @@ export class RemoteDataObject { ASK { ?s nie:hasPart ${sparqlEscapeUri(remoteDataObject)}. - ?fo prov:generated ?s. + ?fo prov:generatedBy ?s. ?fo a ; task:operation . @@ -105,7 +105,9 @@ export class RemoteDataObject { */ static async saveCollection(remotes) { const remotesTriples = remotes - .map((remote) => new RemoteDataObject(remote).toSPARQL()) + .map((remote) => + new RemoteDataObject(remote.uri, remote.address).toSPARQL() + ) .join('\n '); const q = ` PREFIX nfo: @@ -131,12 +133,12 @@ export class RemoteDataObject { } } - constructor({ uri, address }) { + constructor(uri, address) { this.uri = uri; this.address = address; this.uuid = uuid(); this.status = DOWNLOAD_STATUS_READY_TO_BE_CASHED; - this.creator = env.CREATOR; + this.creator = cts.SERVICES.validateSubmission; this.created = new Date(); this.modified = new Date(); } diff --git a/lib/submission-form.js b/lib/submission-form.js index 8ecdda5..09511f0 100644 --- a/lib/submission-form.js +++ b/lib/submission-form.js @@ -1,8 +1,11 @@ -import { uuid, sparqlEscapeUri } from 'mu'; -import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; -import { getFileContent, insertTtlFile, updateTtlFile } from './file-helpers'; -import ForkingStore from 'forking-store'; -import { NamedNode } from 'rdflib'; +import { sparqlEscapeUri } from 'mu'; +import { querySudo, updateSudo } from '@lblod/mu-auth-sudo'; +import * as fil from '../automatic-submission-flow-tools/asfFiles.js'; +import * as sjp from 'sparqljson-parse'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; +import * as uti from '../automatic-submission-flow-tools/utils.js'; +import * as N3 from 'n3'; +const { namedNode } = N3.DataFactory; const FORM_DATA_FILE_TYPE = 'http://data.lblod.gift/concepts/form-data-file-type'; @@ -19,7 +22,7 @@ const FORM_FILE_TYPE = 'http://data.lblod.gift/concepts/form-file-type'; * @return {string} TTL with form description for the given submission document */ export function getFormTtl(submissionDocument) { - return getPartNoLogical(submissionDocument, FORM_FILE_TYPE); + return getPartWithoutLogical(submissionDocument, FORM_FILE_TYPE); } /** @@ -30,21 +33,18 @@ export function getFormTtl(submissionDocument) { * @return {string} TTL with source data for the given submission document */ export async function getSourceTtl(submissionDocument) { - const source = await getHarvestedData(submissionDocument); - const additions = await getAdditions(submissionDocument); - const removals = await getRemovals(submissionDocument); + const sourceContent = await getHarvestedData(submissionDocument); + const additionsContent = await getAdditions(submissionDocument); + const removalsContent = await getRemovals(submissionDocument); - // merge source, additions and removals - const forkingStore = new ForkingStore(); - const graph = new NamedNode(`http://merged-form/graph/${uuid()}`); - forkingStore.loadDataWithAddAndDelGraph( - source || '', - graph, - additions || '', - removals || '', - 'text/turtle' - ); - return forkingStore.serializeDataMergedGraph(graph, 'text/turtle'); + const base = await uti.ttlToStore(sourceContent); + const additions = await uti.ttlToStore(additionsContent); + const removals = await uti.ttlToStore(removalsContent); + + base.removeQuads([...removals]); + base.addQuads([...additions]); + + return uti.storeToTtl(base); } /** @@ -78,9 +78,9 @@ export async function saveFormTriples(submissionDocument, triples) { return file; } -/* - * Private - */ +//////////////////////////////////////////////////////////////////////////////// +// Shared with enrich-submission-service +//////////////////////////////////////////////////////////////////////////////// /** * Get harvested data of a submission document in TTL format. @@ -90,23 +90,25 @@ export async function saveFormTriples(submissionDocument, triples) { * @return {string} TTL with harvested data for the given submission document */ async function getHarvestedData(submissionDocument) { - const result = await query(` + const response = await querySudo(` PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?physicalFile + SELECT DISTINCT ?logicalFile WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?logicalFile . - ?logicalFile dct:type . - ?physicalFile nie:dataSource ?logicalFile . + ${sparqlEscapeUri(submissionDocument)} + dct:source ?logicalFile . + ?logicalFile + dct:type . } - } + } LIMIT 1 `); - if (result.results.bindings.length) { - const file = result.results.bindings[0]['physicalFile'].value; - return await getFileContent(file); + const sparqlJsonParser = new sjp.SparqlJsonParser(); + const parsedResults = sparqlJsonParser.parseJsonResults(response); + + if (parsedResults.length) { + const file = parsedResults[0].logicalFile; + return fil.loadFromLogicalFile(file); } } @@ -140,65 +142,42 @@ function getRemovals(submissionDocument) { * @return {string} Content of the related file */ async function getPart(submissionDocument, fileType) { - const result = await query(` - PREFIX mu: - PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?physicalFile - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?logicalFile . - ?physicalFile nie:dataSource ?logicalFile . - } - - ?logicalFile dct:type ${sparqlEscapeUri(fileType)} . - } - `); - - if (result.results.bindings.length) { - const file = result.results.bindings[0]['physicalFile'].value; - return await getFileContent(file); - } else { - console.log( - `No file of type ${fileType} found for submission document ${submissionDocument}` - ); - return null; - } + const file = await getFileResource(submissionDocument, fileType); + if (file) return fil.loadFromLogicalFile(file); +} +async function getPartWithoutLogical(submissionDocument, fileType) { + const file = await getFileResource(submissionDocument, fileType); + if (file) return fil.loadFromPhysicalFile(file); } /** - * Get the content of a file of the given file type that is related to the given submission document - * This function does not do it properly according to the file service. There is no logical file accompanying the physical file, only the physical file. This is to maintain compatibility with other services in the flow for now. + * Get the file resource in the triplestore of the given file type that is related to the given submission document * * @param {string} submissionDocument URI of the submitted document to get the related file for * @param {string} fileType URI of the type of the related file - * @return {string} Content of the related file + * @return {namedNode|undefined} File full name (path, name and extention) */ -async function getPartNoLogical(submissionDocument, fileType) { - const result = await query(` - PREFIX mu: +async function getFileResource(submissionDocument, fileType) { + const response = await querySudo(` PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?file + SELECT DISTINCT ?logicalFile WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?file . + ${sparqlEscapeUri(submissionDocument)} dct:source ?logicalFile . } - - ?file dct:type ${sparqlEscapeUri(fileType)} . - } + ?logicalFile dct:type ${sparqlEscapeUri(fileType)} . + } LIMIT 1 `); - if (result.results.bindings.length) { - const file = result.results.bindings[0]['file'].value; - return await getFileContent(file); + const sparqlJsonParser = new sjp.SparqlJsonParser(); + const parsedResults = sparqlJsonParser.parseJsonResults(response); + + if (parsedResults.length) { + return parsedResults[0].logicalFile; } else { console.log( - `No file of type ${fileType} found for submission document ${submissionDocument}` + `Part of type ${fileType} for submission document ${submissionDocument} not found` ); - return null; } } @@ -242,43 +221,59 @@ function saveFormData(submissionDocument, content) { * @param {string} fileType URI of the type of the related file */ async function savePart(submissionDocument, content, fileType) { - const result = await query(` - PREFIX mu: + const response = await querySudo(` PREFIX dct: - PREFIX nie: - - SELECT DISTINCT ?file + SELECT ?logicalFile WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} dct:source ?file . + ${sparqlEscapeUri(submissionDocument)} + dct:source ?logicalFile . + ?logicalFile + dct:type ${sparqlEscapeUri(fileType)} . } - ?file dct:type ${sparqlEscapeUri(fileType)} . - } + } LIMIT 0 `); - if (!result.results.bindings.length) { - const logicalFileUri = await insertTtlFile(submissionDocument, content); - await update(` + const sparqlJsonParser = new sjp.SparqlJsonParser(); + const parsedResults = sparqlJsonParser.parseJsonResults(response); + + if (!parsedResults.length) { + const graphResponse = await querySudo(` + PREFIX ext: + SELECT ?g WHERE { + GRAPH ?g { + ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . + } + } LIMIT 1 + `); + const graphResults = sparqlJsonParser.parseJsonResults(graphResponse); + const graph = graphResults[0]?.g; + const { logicalFile } = await fil.createFromContent( + content, + namedNode(cts.SERVICES.enrichSubmission), + graph + ); + await updateSudo(` PREFIX dct: PREFIX ext: - INSERT { GRAPH ?g { ${sparqlEscapeUri(submissionDocument)} - dct:source ${sparqlEscapeUri(logicalFileUri)} . - ${sparqlEscapeUri(logicalFileUri)} + dct:source ${sparqlEscapeUri(logicalFile.value)} . + ${sparqlEscapeUri(logicalFile.value)} dct:type ${sparqlEscapeUri(fileType)} . } } WHERE { GRAPH ?g { - ${sparqlEscapeUri(submissionDocument)} a ext:SubmissionDocument . + ${sparqlEscapeUri(submissionDocument)} + a ext:SubmissionDocument . } } `); - return logicalFileUri; + return logicalFile.value; } else { - const file = result.results.bindings[0]['file'].value; - await updateTtlFile(submissionDocument, file, content); - return file; + const logicalFile = parsedResults[0].logicalFile; + await fil.updateContentForLogicalFile(logicalFile, content); + return logicalFile; } } diff --git a/lib/submission-task.js b/lib/submission-task.js deleted file mode 100644 index 8780fda..0000000 --- a/lib/submission-task.js +++ /dev/null @@ -1,78 +0,0 @@ -import * as mu from 'mu'; -import * as mas from '@lblod/mu-auth-sudo'; -import * as env from '../env.js'; - -/** - * Updates the state of the given task to the specified status with potential error message or resultcontainer file - * - * @param string taskUri URI of the task - * @param string status URI of the new status - * @param string or undefined URI of the error that needs to be attached - */ -export async function updateTaskStatus( - taskUri, - status, - errorUri, - extraStatus, - logicalFileUri -) { - const taskUriSparql = mu.sparqlEscapeUri(taskUri); - const nowSparql = mu.sparqlEscapeDateTime(new Date().toISOString()); - const hasError = errorUri && status === env.TASK_FAILURE_STATUS; - - let resultContainerTriples = ''; - let resultContainerUuid = ''; - if (logicalFileUri) { - resultContainerUuid = mu.uuid(); - resultContainerTriples = ` - asj:${resultContainerUuid} - a nfo:DataContainer ; - mu:uuid ${mu.sparqlEscapeString(resultContainerUuid)} ; - task:hasFile ${mu.sparqlEscapeUri(logicalFileUri)} . - `; - } - const resultsContainerLink = resultContainerUuid - ? `task:resultsContainer asj:${resultContainerUuid} ;` - : ''; - - //TODO This triple does not do anything? Where is inputContainer defined? - //Searched through toezicht-flattened-form-data-generator, and could not find ext:additionalStatus anywhere, so this can be removed later. - const extraStatusTriple = - status === env.TASK_SUCCESS_STATUS && extraStatus - ? '?inputContainer ext:additionalStatus ' + - mu.sparqlEscapeUri(extraStatus) + - ' .' - : ''; - - const statusUpdateQuery = ` - ${env.PREFIXES} - DELETE { - GRAPH ?g { - ${taskUriSparql} - adms:status ?oldStatus ; - dct:modified ?oldModified . - } - } - INSERT { - GRAPH ?g { - ${taskUriSparql} - adms:status ${mu.sparqlEscapeUri(status)} ; - ${resultsContainerLink} - ${hasError ? `task:error ${mu.sparqlEscapeUri(errorUri)} ;` : ''} - dct:modified ${nowSparql} . - - ${extraStatusTriple} - - ${resultContainerTriples} - } - } - WHERE { - GRAPH ?g { - ${taskUriSparql} - adms:status ?oldStatus ; - dct:modified ?oldModified . - } - } - `; - await mas.updateSudo(statusUpdateQuery); -} diff --git a/lib/submission.js b/lib/submission.js index a053e58..0241459 100644 --- a/lib/submission.js +++ b/lib/submission.js @@ -1,5 +1,5 @@ -import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu'; -import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; +import { sparqlEscapeUri, sparqlEscapeDateTime } from 'mu'; +import { updateSudo as update } from '@lblod/mu-auth-sudo'; import FormBuilder from './form-builder'; import { getFormTtl, @@ -10,16 +10,13 @@ import { } from './submission-form'; import { RemoteDataObject } from './remote-data-object'; import * as env from '../env.js'; - -export const CONCEPT_STATUS = - 'http://lblod.data.gift/concepts/79a52da4-f491-4e2f-9374-89a13cde8ecd'; -export const SUBMITABLE_STATUS = - 'http://lblod.data.gift/concepts/f6330856-e261-430f-b949-8e510d20d0ff'; -export const SENT_STATUS = - 'http://lblod.data.gift/concepts/9bd8d86d-bb10-4456-a84e-91e9507c374c'; +import * as cts from '../automatic-submission-flow-tools/constants.js'; +import * as smt from '../automatic-submission-flow-tools/asfSubmissions.js'; +import * as N3 from 'n3'; +const { namedNode, literal } = N3.DataFactory; export default class Submission { - constructor({ uri, status, submittedResource }) { + constructor(uri, status, submittedResource) { this.uri = uri; this.status = status; this.submittedResource = submittedResource; // submission document @@ -27,8 +24,7 @@ export default class Submission { async updateStatus(status) { await update(` - PREFIX adms: - + ${cts.SPARQL_PREFIXES} DELETE { GRAPH ?g { ${sparqlEscapeUri(this.uri)} adms:status ?status . @@ -49,7 +45,7 @@ export default class Submission { /** * Updates the additions and removals and processes the form */ - async update({ additions, removals }) { + async update(additions, removals) { await updateDocument(this.submittedResource, { additions, removals }); await this.process(); } @@ -62,12 +58,12 @@ export default class Submission { const formTtl = await getFormTtl(this.submittedResource); const metaTtl = await getMetaTtl(this.submittedResource); const sourceTtl = await getSourceTtl(this.submittedResource); - const formBuilder = new FormBuilder({ - submittedResource: this.submittedResource, + const formBuilder = new FormBuilder( + this.submittedResource, formTtl, sourceTtl, - metaTtl, - }); + metaTtl + ); const triples = formBuilder.build().data(); if (!triples.length) { @@ -81,18 +77,19 @@ export default class Submission { `Form for submitted resource ${this.submittedResource} is valid: ${isValid}` ); - const currentStatus = await getSubmissionStatus(this.uri); + const submissionInfo = await smt.getSubmissionInfo(namedNode(this.uri)); + const currentStatus = submissionInfo?.status?.value; if (!currentStatus) throw new Error(`Submission <${this.uri}> doesn't have a status`); let targetStatus = null; - if (currentStatus === SUBMITABLE_STATUS) { + if (currentStatus === env.SUBMITABLE_STATUS) { if (!isValid) { console.log( `Resetting status of submission ${this.uri} to concept since it's invalid` ); - targetStatus = CONCEPT_STATUS; + targetStatus = env.CONCEPT_STATUS; } else { console.log( `Updating status of submission ${this.uri} to sent state since it's valid` @@ -100,7 +97,7 @@ export default class Submission { await this.submit(formBuilder.form.value, triples); // NOTE find and save/update remote-data-objects await RemoteDataObject.process(triples); - targetStatus = SENT_STATUS; + targetStatus = env.SENT_STATUS; } } @@ -162,77 +159,25 @@ export default class Submission { } export async function getSubmissionByTask(taskUri) { - const response = await query(` - ${env.PREFIXES} - SELECT ?submission ?submissionDocument ?status - WHERE { - GRAPH ?g { - ${sparqlEscapeUri(taskUri)} - a task:Task ; - dct:isPartOf ?job . - ?job prov:generated ?submission . - ?submission - dct:subject ?submissionDocument ; - adms:status ?status . - } - } LIMIT 1 - `); - - const bindings = response?.results?.bindings; - if (bindings && bindings.length > 0) { - const binding = bindings[0]; - return new Submission({ - uri: binding.submission.value, - status: binding.status.value, - submittedResource: binding.submissionDocument.value, - }); - } + const submissionInfo = await smt.getSubmissionInfoFromTask( + namedNode(taskUri) + ); + if (submissionInfo) + return new Submission( + submissionInfo.submission.value, + submissionInfo.status.value, + submissionInfo.submittedDocument.value + ); } export async function getSubmissionBySubmissionDocument(uuid) { - const result = await query(` - PREFIX nie: - PREFIX prov: - PREFIX melding: - PREFIX adms: - PREFIX dct: - PREFIX mu: - - SELECT ?submission ?submissionDocument ?status - WHERE { - GRAPH ?g { - ?submissionDocument mu:uuid ${sparqlEscapeString(uuid)} . - ?submission - dct:subject ?submissionDocument ; - adms:status ?status . - } - } LIMIT 1 - `); - - if (result.results.bindings.length) { - const binding = result.results.bindings[0]; - return new Submission({ - uri: binding['submission'].value, - status: binding['status'].value, - submittedResource: binding['submissionDocument'].value, - }); - } else { - return null; - } -} - -export async function getSubmissionStatus(submissionUri) { - const result = await query(` - PREFIX adms: - - SELECT ?status - WHERE { ${sparqlEscapeUri(submissionUri)} adms:status ?status . } - LIMIT 1 -`); - - if (result.results.bindings.length) { - return result.results.bindings[0]['status'].value; - } else { - return null; - } + const submissionInfo = await smt.getSubmissionInfoFromSubmissionDocumentId( + literal(uuid) + ); + if (submissionInfo) + return new Submission( + submissionInfo.submission.value, + submissionInfo.status.value, + submissionInfo.submittedDocument.value + ); } diff --git a/lib/utils.js b/lib/utils.js deleted file mode 100644 index 3845a79..0000000 --- a/lib/utils.js +++ /dev/null @@ -1,43 +0,0 @@ -import * as mas from '@lblod/mu-auth-sudo'; -import * as mu from 'mu'; -import * as env from '../env.js'; - -export async function saveError({ message, detail, reference }) { - if (!message) throw 'Error needs a message describing what went wrong.'; - const id = mu.uuid(); - const uri = `http://data.lblod.info/errors/${id}`; - const referenceTriple = reference - ? `dct:references ${mu.sparqlEscapeUri(reference)} ;` - : ''; - const detailTriple = detail - ? `oslc:largePreview ${mu.sparqlEscapeString(detail)} ;` - : ''; - const q = ` - PREFIX mu: - PREFIX oslc: - PREFIX dct: - PREFIX xsd: - - INSERT DATA { - GRAPH { - ${mu.sparqlEscapeUri(uri)} - a oslc:Error ; - mu:uuid ${mu.sparqlEscapeString(id)} ; - dct:subject ${mu.sparqlEscapeString('Automatic Submission Service')} ; - oslc:message ${mu.sparqlEscapeString(message)} ; - dct:created ${mu.sparqlEscapeDateTime(new Date().toISOString())} ; - ${referenceTriple} - ${detailTriple} - dct:creator ${mu.sparqlEscapeUri(env.CREATOR)} . - } - } - `; - try { - await mas.updateSudo(q); - return uri; - } catch (e) { - console.warn( - `[WARN] Something went wrong while trying to store an error.\nMessage: ${e}\nQuery: ${q}` - ); - } -} diff --git a/package.json b/package.json index fccc92c..403deb2 100644 --- a/package.json +++ b/package.json @@ -12,10 +12,11 @@ "@lblod/mu-auth-sudo": "^0.6.0", "@lblod/submission-form-helpers": "^1.3.0", "body-parser": "1.20.0", - "forking-store": "^1.0.0", - "fs-extra": "^8.1.0", - "lodash.flatten": "^4.4.0", - "moment": "^2.24.0", + "automatic-submission-flow-tools": "file:automatic-submission-flow-tools", + "n3": "^1.16.2", + "rdf-string-ttl": "github:benjay10/rdf-string-ttl.js", + "sparqljson-parse": "^2.1.1", + "uuid": "^9.0.0", "rdflib": "1.1.0" }, "devDependencies": {