From a0aeaaaf7c00cfd3e22c312d01b9be7c45aa6653 Mon Sep 17 00:00:00 2001 From: Adler Faulkner Date: Tue, 19 Sep 2023 12:13:22 -0700 Subject: [PATCH 1/2] feat(verb-config): add inputFiles and functions to verbConfig --- package-lock.json | 14 +-- package.json | 2 +- src/SklEngine.ts | 106 ++++++++++++++---- src/SklEngineOptions.ts | 4 + src/storage/BaseQueryAdapterOptions.ts | 4 - src/util/Types.ts | 11 ++ .../assets/schemas/json-file-data-source.json | 6 +- test/unit/SklEngine.test.ts | 2 +- 8 files changed, 109 insertions(+), 40 deletions(-) diff --git a/package-lock.json b/package-lock.json index ed6fabb..08fd134 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "BSD-4-Clause", "dependencies": { "@comake/openapi-operation-executor": "^0.11.1", - "@comake/rmlmapper-js": "^0.4.0", + "@comake/rmlmapper-js": "^0.5.1", "@rdfjs/data-model": "^1.3.0", "jsonld": "^8.1.0", "jsonpath-plus": "^7.2.0", @@ -733,9 +733,9 @@ } }, "node_modules/@comake/rmlmapper-js": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@comake/rmlmapper-js/-/rmlmapper-js-0.4.0.tgz", - "integrity": "sha512-a5loYod0LwJaRhzomWJ1VGUHnoWpBG9McT3sibfHL8aUTh2tqhqKq3++qTsV7z1wCR9TqitdIxTIhBeXdkrHxQ==", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@comake/rmlmapper-js/-/rmlmapper-js-0.5.1.tgz", + "integrity": "sha512-wBJpOfM+v4MgbwVw3Y7QwsbsyEncx8rVYXQeUcwaOFX1Gq5ysY15tfvhtwGihgBTZ+AM72wbFbeYOd+vik4gIw==", "dependencies": { "@xmldom/xmldom": "^0.8.2", "csvjson": "^5.1.0", @@ -10253,9 +10253,9 @@ } }, "@comake/rmlmapper-js": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@comake/rmlmapper-js/-/rmlmapper-js-0.4.0.tgz", - "integrity": "sha512-a5loYod0LwJaRhzomWJ1VGUHnoWpBG9McT3sibfHL8aUTh2tqhqKq3++qTsV7z1wCR9TqitdIxTIhBeXdkrHxQ==", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@comake/rmlmapper-js/-/rmlmapper-js-0.5.1.tgz", + "integrity": "sha512-wBJpOfM+v4MgbwVw3Y7QwsbsyEncx8rVYXQeUcwaOFX1Gq5ysY15tfvhtwGihgBTZ+AM72wbFbeYOd+vik4gIw==", "requires": { "@xmldom/xmldom": "^0.8.2", "csvjson": "^5.1.0", diff --git a/package.json b/package.json index 777ba7b..cb53c3f 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ }, "dependencies": { "@comake/openapi-operation-executor": "^0.11.1", - "@comake/rmlmapper-js": "^0.4.0", + "@comake/rmlmapper-js": "^0.5.1", "@rdfjs/data-model": "^1.3.0", "jsonld": "^8.1.0", "jsonpath-plus": "^7.2.0", diff --git a/src/SklEngine.ts b/src/SklEngine.ts index f27e42f..06e8df3 100644 --- a/src/SklEngine.ts +++ b/src/SklEngine.ts @@ -55,8 +55,8 @@ export type VerbInterface = Record; export type MappingResponseOption = T extends true ? JSONObject : NodeObject; export class SKLEngine { - private readonly mapper: Mapper; private readonly adapter: QueryAdapter; + private readonly functions?: Record any>; private readonly inputFiles?: Record; private readonly globalCallbacks?: Callbacks; private readonly disableValidation?: boolean; @@ -77,7 +77,7 @@ export class SKLEngine { this.disableValidation = options.disableValidation; this.globalCallbacks = options.callbacks; this.inputFiles = options.inputFiles; - this.mapper = new Mapper({ functions: options.functions }); + this.functions = options.functions; // eslint-disable-next-line func-style const getVerbHandler = (getTarget: VerbInterface, property: string): VerbHandler => @@ -164,8 +164,14 @@ export class SKLEngine { args: JSONObject, mapping: OrArray, frame?: Record, + verbConfig?: VerbConfig, ): Promise { - return await this.mapper.apply(args, mapping, frame ?? {}); + const functions = { + ...this.functions, + ...verbConfig?.functions, + }; + const mapper = new Mapper({ functions }); + return await mapper.apply(args, mapping, frame ?? {}); } public async executeTrigger( @@ -176,6 +182,7 @@ export class SKLEngine { const verbArgs = await this.performParameterMappingOnArgsIfDefined( payload, triggerToVerbMapping, + undefined, false, ); const verbId = await this.performVerbMappingWithArgs(payload, triggerToVerbMapping); @@ -223,6 +230,7 @@ export class SKLEngine { verbReturnValue = await this.performReturnValueMappingWithFrame( verbArgs, verb as MappingWithReturnValueMapping, + verbConfig, ); } else if (SKL.series in verb) { verbReturnValue = await this.executeSeriesVerb(verb, verbArgs, verbConfig); @@ -296,10 +304,15 @@ export class SKLEngine { args: JSONObject, verbConfig?: VerbConfig, ): Promise> { - args = await this.addPreProcessingMappingToArgs(verbMapping, args); - const verbId = await this.performVerbMappingWithArgs(args, verbMapping); + args = await this.addPreProcessingMappingToArgs(verbMapping, args, verbConfig); + const verbId = await this.performVerbMappingWithArgs(args, verbMapping, verbConfig); if (verbId) { - const verbArgs = await this.performParameterMappingOnArgsIfDefined({ ...args, verbId }, verbMapping, false); + const verbArgs = await this.performParameterMappingOnArgsIfDefined( + { ...args, verbId }, + verbMapping, + verbConfig, + false, + ); if (verbId === SKL_ENGINE.update) { await this.updateEntityFromVerbArgs(verbArgs); return {}; @@ -327,6 +340,7 @@ export class SKLEngine { return await this.performReturnValueMappingWithFrame( returnValue as JSONObject, verbMapping as MappingWithReturnValueMapping, + verbConfig, ); } return returnValue; @@ -334,12 +348,17 @@ export class SKLEngine { return {}; } - private async addPreProcessingMappingToArgs(verbMapping: VerbMapping, args: JSONObject): Promise { + private async addPreProcessingMappingToArgs( + verbMapping: VerbMapping, + args: JSONObject, + verbConfig?: VerbConfig, + ): Promise { if (SKL.preProcessingMapping in verbMapping) { const preMappingArgs = await this.performMapping( args, verbMapping[SKL.preProcessingMapping] as NodeObject, getValueIfDefined(verbMapping[SKL.preProcessingMappingFrame]), + verbConfig, ); return { ...args, preProcessedParameters: preMappingArgs as JSONObject }; } @@ -422,9 +441,10 @@ export class SKLEngine { const operationArgs = await this.performParameterMappingOnArgsIfDefined( args, mapping as MappingWithParameterMapping, + verbConfig, ); - const operationInfo = await this.performOperationMappingWithArgs(args, mapping); - const rawReturnValue = await this.performOperation(operationInfo, operationArgs, account); + const operationInfo = await this.performOperationMappingWithArgs(args, mapping, verbConfig); + const rawReturnValue = await this.performOperation(operationInfo, operationArgs, account, undefined, verbConfig); if (operationInfo[SKL.schemeName] && rawReturnValue.data.authorizationUrl) { return { '@type': '@json', @@ -436,6 +456,7 @@ export class SKLEngine { const mappedReturnValue = await this.performReturnValueMappingWithFrame( rawReturnValue, mapping as MappingWithReturnValueMapping, + verbConfig, verb, ); if (shouldValidate) { @@ -457,11 +478,20 @@ export class SKLEngine { private async performOperationMappingWithArgs( args: JSONObject, mapping: MappingWithOperationMapping, + verbConfig?: VerbConfig, ): Promise { if (mapping[SKL.operationId]) { return { [SKL.operationId]: mapping[SKL.operationId] }; } - return await this.performMapping(args, mapping[SKL.operationMapping] as OrArray); + if (mapping[SKL.dataSource]) { + return { [SKL.dataSource]: mapping[SKL.dataSource] }; + } + return await this.performMapping( + args, + mapping[SKL.operationMapping] as OrArray, + undefined, + verbConfig, + ); } private async performOperation( @@ -469,6 +499,7 @@ export class SKLEngine { operationArgs: JSONObject, account: Entity, securityCredentials?: Entity, + verbConfig?: VerbConfig, ): Promise { if (operationInfo[SKL.schemeName]) { return await this.performOauthSecuritySchemeStageWithCredentials( @@ -480,7 +511,8 @@ export class SKLEngine { } if (operationInfo[SKL.dataSource]) { return await this.getDataFromDataSource( - getValueIfDefined(operationInfo[SKL.dataSource])!, + getIdFromNodeObjectIfDefined(operationInfo[SKL.dataSource] as string | ReferenceNodeObject)!, + verbConfig, ); } if (operationInfo[SKL.operationId]) { @@ -516,6 +548,7 @@ export class SKLEngine { private async performReturnValueMappingWithFrame( returnValue: JSONObject, mapping: MappingWithReturnValueMapping, + verbConfig?: VerbConfig, verb?: Entity, ): Promise { return await this.performMapping( @@ -525,12 +558,14 @@ export class SKLEngine { ...getValueIfDefined(verb?.[SKL.returnValueFrame]), ...getValueIfDefined(mapping[SKL.returnValueFrame]), }, + verbConfig, ); } private async performParameterMappingOnArgsIfDefined( args: JSONObject, mapping: Partial | Partial, + verbConfig?: VerbConfig, convertToJsonDeep = true, ): Promise> { if (SKL.parameterReference in mapping) { @@ -542,6 +577,7 @@ export class SKLEngine { args, (mapping as MappingWithParameterMapping)[SKL.parameterMapping]!, getValueIfDefined(mapping[SKL.parameterMappingFrame]), + verbConfig, ); return toJSON(mappedData, convertToJsonDeep); } @@ -586,10 +622,15 @@ export class SKLEngine { private async executeNounMappingVerb(verb: Entity, args: JSONObject, verbConfig?: VerbConfig): Promise { const mapping = await this.findVerbNounMapping(verb['@id'], args.noun as string); if (mapping[SKL.returnValueMapping]) { - return await this.performReturnValueMappingWithFrame(args, mapping as MappingWithReturnValueMapping, verb); + return await this.performReturnValueMappingWithFrame( + args, + mapping as MappingWithReturnValueMapping, + verbConfig, + verb, + ); } - const verbArgs = await this.performParameterMappingOnArgsIfDefined(args, mapping, false); - const verbId = await this.performVerbMappingWithArgs(args, mapping); + const verbArgs = await this.performParameterMappingOnArgsIfDefined(args, mapping, verbConfig, false); + const verbId = await this.performVerbMappingWithArgs(args, mapping, verbConfig); const mappedVerb = (await this.findBy({ id: verbId })) as Verb; this.globalCallbacks?.onVerbStart?.(verb['@id'], verbArgs); @@ -617,11 +658,17 @@ export class SKLEngine { private async performVerbMappingWithArgs( args: JSONObject, mapping: MappingWithVerbMapping, + verbConfig?: VerbConfig, ): Promise { if (mapping[SKL.verbId]) { return getValueIfDefined(mapping[SKL.verbId])!; } - const verbInfoJsonLd = await this.performMapping(args, mapping[SKL.verbMapping] as NodeObject); + const verbInfoJsonLd = await this.performMapping( + args, + mapping[SKL.verbMapping] as NodeObject, + undefined, + verbConfig, + ); return getValueIfDefined(verbInfoJsonLd[SKL.verbId])!; } @@ -693,6 +740,7 @@ export class SKLEngine { securityCredentials: Entity, integrationId: string, account: Entity, + verbConfig?: VerbConfig, ): Promise { const getOauthTokenVerb = (await this.findBy({ type: SKL.Verb, [RDFS.label]: 'getOauthTokens' })) as Verb; const mapping = await this.findVerbIntegrationMapping(getOauthTokenVerb['@id'], integrationId); @@ -702,17 +750,20 @@ export class SKLEngine { jwtBearerOptions: getValueIfDefined(securityCredentials[SKL.jwtBearerOptions])!, }, mapping, + verbConfig, ); - const operationInfoJsonLd = await this.performOperationMappingWithArgs({}, mapping); + const operationInfoJsonLd = await this.performOperationMappingWithArgs({}, mapping, verbConfig); const rawReturnValue = await this.performOperation( operationInfoJsonLd, operationArgs, account, securityCredentials, + verbConfig, ); const mappedReturnValue = await this.performReturnValueMappingWithFrame( rawReturnValue, mapping as MappingWithReturnValueMapping, + verbConfig, getOauthTokenVerb, ); await this.assertVerbReturnValueMatchesReturnTypeSchema(mappedReturnValue, getOauthTokenVerb); @@ -833,27 +884,34 @@ export class SKLEngine { return this.axiosResponseAndParamsToOperationResponse(response, operationParameters); } - private async getDataFromDataSource(dataSourceId: string): Promise { + private async getDataFromDataSource(dataSourceId: string, verbConfig?: VerbConfig): Promise { const dataSource = await this.findBy({ id: dataSourceId }); if (dataSource['@type'] === SKL.JsonDataSource) { - const data = this.getDataFromJsonDataSource(dataSource); + const data = this.getDataFromJsonDataSource(dataSource, verbConfig); return { data, operationParameters: {}}; } throw new Error(`DataSource type ${dataSource['@type']} is not supported.`); } - private getDataFromJsonDataSource(dataSource: NodeObject): JSONObject { + private getDataFromJsonDataSource(dataSource: NodeObject, verbConfig?: VerbConfig): JSONObject { if (dataSource[SKL.source]) { const sourceValue = getValueIfDefined(dataSource[SKL.source])!; - return this.getJsonDataFromSource(sourceValue); + return this.getJsonDataFromSource(sourceValue, verbConfig); } return getValueIfDefined(dataSource[SKL.data])!; } - private getJsonDataFromSource(source: string): JSONObject { - if (this.inputFiles && source in this.inputFiles) { - const file = this.inputFiles[source]; - return JSON.parse(file); + private getJsonDataFromSource(source: string, verbConfig?: VerbConfig): JSONObject { + const inputFiles = { + ...this.inputFiles, + ...verbConfig?.inputFiles, + }; + if (inputFiles && source in inputFiles) { + const file = inputFiles[source]; + if (typeof file === 'string') { + return JSON.parse(file); + } + return file; } // eslint-disable-next-line unicorn/expiring-todo-comments // TODO add support for remote sources diff --git a/src/SklEngineOptions.ts b/src/SklEngineOptions.ts index 74ff1cb..036d10b 100644 --- a/src/SklEngineOptions.ts +++ b/src/SklEngineOptions.ts @@ -19,4 +19,8 @@ export type SklEngineOptions = SklEngineStorageOptions & { * When true, disables validation of verb parameters and return values according to schemas */ readonly disableValidation?: boolean; + /** + * An object containing files keyed on their title that can be used in mappings. + */ + readonly inputFiles?: Record; }; diff --git a/src/storage/BaseQueryAdapterOptions.ts b/src/storage/BaseQueryAdapterOptions.ts index 20875de..b47557c 100644 --- a/src/storage/BaseQueryAdapterOptions.ts +++ b/src/storage/BaseQueryAdapterOptions.ts @@ -5,10 +5,6 @@ export interface BaseQueryAdapterOptions { * Query Adapter type. This value is required. */ readonly type: QueryAdapterType; - /** - * An object containing files keyed on their title that can be used in mappings. - */ - readonly inputFiles?: Record; /** * Whether to set Dublic Core created and modified timestamps on saved entities. Defaults to false. */ diff --git a/src/util/Types.ts b/src/util/Types.ts index 58b1ac5..5089ee5 100644 --- a/src/util/Types.ts +++ b/src/util/Types.ts @@ -137,6 +137,7 @@ export interface OperationResponse extends JSONObject { export interface VerbConfig { /** * Callbacks to execute upon events. + * If global callbacks are provided, both are executed. */ callbacks?: Callbacks; /** @@ -144,4 +145,14 @@ export interface VerbConfig { * return values according to schemas. Overrides the global setting. */ readonly disableValidation?: boolean; + /** + * An object containing files keyed on their title that can be used in mappings. + * Merged with the global setting. The verb config taking prededence in the case of overlapping names. + */ + readonly inputFiles?: Record; + /** + * Manually defined functions which can be used in mappings. + * Merged with the global setting. The verb config taking prededence in the case of overlapping names. + */ + readonly functions?: Record any>; } diff --git a/test/assets/schemas/json-file-data-source.json b/test/assets/schemas/json-file-data-source.json index 093f862..2290521 100644 --- a/test/assets/schemas/json-file-data-source.json +++ b/test/assets/schemas/json-file-data-source.json @@ -58,10 +58,10 @@ "label": "JsonSource" }, { - "@id": "https://example.com/data/JsonSourceDataSource", + "@id": "https://example.com/data/JsonFileDataSource", "@type": "https://standardknowledge.com/ontologies/core/JsonDataSource", "integration": "https://example.com/integrations/JsonSource", - "https://standardknowledge.com/ontologies/core/data": { + "https://standardknowledge.com/ontologies/core/data": { ".tag": "file", "client_modified": "2015-05-12T15:50:38Z", "content_hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -94,7 +94,7 @@ "@type": "rr:PredicateObjectMap", "rr:objectMap": { "@type": "rr:ObjectMap", - "rr:constant": "https://example.com/data/JsonSourceDataSource" + "rr:constant": "https://example.com/data/JsonFileDataSource" }, "rr:predicate": "skl:dataSource" } diff --git a/test/unit/SklEngine.test.ts b/test/unit/SklEngine.test.ts index ec680db..1ab4076 100644 --- a/test/unit/SklEngine.test.ts +++ b/test/unit/SklEngine.test.ts @@ -886,7 +886,7 @@ describe('SKLEngine', (): void => { './test/assets/schemas/json-file-data-source.json', ]); schemas = schemas.map((schemaItem: any): any => { - if (schemaItem['@id'] === 'https://example.com/data/JsonSourceDataSource') { + if (schemaItem['@id'] === 'https://example.com/data/JsonFileDataSource') { schemaItem['@type'] = 'https://standardknowledge.com/ontologies/core/CsvDataSource'; } return schemaItem; From d53c222b3992d4bec24f4e51d78087bd92f8fba8 Mon Sep 17 00:00:00 2001 From: Adler Faulkner Date: Tue, 19 Sep 2023 12:16:46 -0700 Subject: [PATCH 2/2] chore: bump version number --- package-lock.json | 4 ++-- package.json | 2 +- test/deploy/package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 08fd134..9a1676a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@comake/skl-js-engine", - "version": "0.13.4", + "version": "0.14.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@comake/skl-js-engine", - "version": "0.13.4", + "version": "0.14.0", "license": "BSD-4-Clause", "dependencies": { "@comake/openapi-operation-executor": "^0.11.1", diff --git a/package.json b/package.json index cb53c3f..36c104c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@comake/skl-js-engine", - "version": "0.13.4", + "version": "0.14.0", "description": "Standard Knowledge Language Javascript Engine", "keywords": [ "skl", diff --git a/test/deploy/package.json b/test/deploy/package.json index 4b8e4dd..6c2faee 100644 --- a/test/deploy/package.json +++ b/test/deploy/package.json @@ -7,7 +7,7 @@ }, "main": "./dist/index.js", "dependencies": { - "@comake/skl-js-engine": "file:./comake-skl-js-engine-0.13.4.tgz", + "@comake/skl-js-engine": "file:./comake-skl-js-engine-0.14.0.tgz", "jsonld": "^6.0.0" }, "devDependencies": {