diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..55712c1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib" +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1747c03..fe4e36f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@comake/skl-js-engine", - "version": "0.21.0", + "version": "0.22.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@comake/skl-js-engine", - "version": "0.21.0", + "version": "0.22.0", "license": "BSD-4-Clause", "dependencies": { "@comake/openapi-operation-executor": "^0.11.1", @@ -50,7 +50,7 @@ "eslint-plugin-tsdoc": "^0.2.16", "eslint-plugin-unused-imports": "^2.0.0", "fs": "^0.0.1-security", - "husky": "^8.0.0", + "husky": "^8.0.3", "jest": "^29.5.0", "jsdom": "^20.0.0", "ts-jest": "^29.1.0", @@ -7797,10 +7797,11 @@ } }, "node_modules/husky": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.1.tgz", - "integrity": "sha512-xs7/chUH/CKdOCs7Zy0Aev9e/dKOMZf3K1Az1nar3tzlv0jfqnYtu235bstsWTmXOR0EfINrPa97yy4Lz6RiKw==", + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", + "integrity": "sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==", "dev": true, + "license": "MIT", "bin": { "husky": "lib/bin.js" }, @@ -19236,9 +19237,9 @@ "dev": true }, "husky": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.1.tgz", - "integrity": "sha512-xs7/chUH/CKdOCs7Zy0Aev9e/dKOMZf3K1Az1nar3tzlv0jfqnYtu235bstsWTmXOR0EfINrPa97yy4Lz6RiKw==", + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", + "integrity": "sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg==", "dev": true }, "iconv-lite": { diff --git a/package.json b/package.json index c4a7980..5f85ef8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@comake/skl-js-engine", - "version": "0.21.0", + "version": "0.22.0", "description": "Standard Knowledge Language Javascript Engine", "keywords": [ "skl", @@ -83,7 +83,7 @@ "eslint-plugin-tsdoc": "^0.2.16", "eslint-plugin-unused-imports": "^2.0.0", "fs": "^0.0.1-security", - "husky": "^8.0.0", + "husky": "^8.0.3", "jest": "^29.5.0", "jsdom": "^20.0.0", "ts-jest": "^29.1.0", diff --git a/src/SklEngine.ts b/src/SklEngine.ts index 17393b2..8ea4506 100644 --- a/src/SklEngine.ts +++ b/src/SklEngine.ts @@ -53,6 +53,7 @@ import { ensureArray, } from './util/Util'; import { SKL, SHACL, RDFS, SKL_ENGINE, XSD, RDF } from './util/Vocabularies'; +import { GroupByOptions, GroupByResponse } from './storage/GroupOptionTypes'; export type VerbHandler = = OrArray>( params: JSONObject, @@ -131,6 +132,10 @@ export class SKLEngine { return await this.queryAdapter.findAll(options); } + public async groupBy(options: GroupByOptions): Promise { + return await this.queryAdapter.groupBy(options); + } + public async findAllBy(where: FindOptionsWhere): Promise { return await this.queryAdapter.findAllBy(where); } diff --git a/src/storage/GroupOptionTypes.ts b/src/storage/GroupOptionTypes.ts new file mode 100644 index 0000000..001b99e --- /dev/null +++ b/src/storage/GroupOptionTypes.ts @@ -0,0 +1,32 @@ +import { FindOptionsWhere } from "./FindOptionsTypes"; + +// Add these types at the top of the file +export interface GroupByOptions { + where?: FindOptionsWhere; + groupBy?: string[]; + dateRange?: { + start: string; + end: string; + }; + dateGrouping?: "month" | "day"; + limit?: number; + offset?: number; +} + +export interface GroupResult { + group: Record; + count: number; + entityIds: string[]; +} + +export interface GroupByResponse { + results: GroupResult[]; + meta: { + totalCount: number; + dateRange?: { + start: string; + end: string; + }; + groupings: string[]; + }; +} diff --git a/src/storage/query-adapter/QueryAdapter.ts b/src/storage/query-adapter/QueryAdapter.ts index 76f663e..c357d91 100644 --- a/src/storage/query-adapter/QueryAdapter.ts +++ b/src/storage/query-adapter/QueryAdapter.ts @@ -9,6 +9,7 @@ import type { FindOneOptions, FindOptionsWhere, } from '../FindOptionsTypes'; +import { GroupByOptions, GroupByResponse } from '../GroupOptionTypes'; export type RawQueryResult = Record; @@ -102,4 +103,8 @@ export interface QueryAdapter { * Removes all entities from the database. */ destroyAll(): Promise; + /** + * Groups entities by a given options. + */ + groupBy(options: GroupByOptions): Promise; } diff --git a/src/storage/query-adapter/sparql/SparqlQueryAdapter.ts b/src/storage/query-adapter/sparql/SparqlQueryAdapter.ts index 6d479b4..115436d 100644 --- a/src/storage/query-adapter/sparql/SparqlQueryAdapter.ts +++ b/src/storage/query-adapter/sparql/SparqlQueryAdapter.ts @@ -37,6 +37,7 @@ import type { QueryExecutor } from './query-executor/SparqlQueryExecutor'; import type { SparqlQueryAdapterOptions } from './SparqlQueryAdapterOptions'; import { SparqlQueryBuilder } from './SparqlQueryBuilder'; import { SparqlUpdateBuilder } from './SparqlUpdateBuilder'; +import { GroupByOptions, GroupByResponse, GroupResult } from '../../GroupOptionTypes'; /** * A {@link QueryAdapter} that stores data in a database through a sparql endpoint. @@ -213,6 +214,58 @@ export class SparqlQueryAdapter implements QueryAdapter { return entityOrEntities; } + public async groupBy(options: GroupByOptions): Promise { + const queryBuilder = new SparqlQueryBuilder(); + const { query: selectQuery, variableMapping } = await queryBuilder.buildGroupByQuery(options); + const results = await this.queryExecutor.executeSparqlSelectAndGetData( + selectQuery + ); + + // Create reverse mapping from path to variable name + const reverseMapping = Object.entries(variableMapping).reduce((acc, [varName, path]) => { + acc[path] = varName; + return acc; + }, {} as Record); + + // Transform results + const groupResults: GroupResult[] = results.map((result) => { + const group: Record = {}; + + options.groupBy?.forEach((path) => { + const varName = reverseMapping[path]; + if (!varName) { + throw new Error(`No variable mapping found for path: ${path}`); + } + const value = result[varName].value; + // Try to convert to number if possible + group[path] = isNaN(Number(value)) ? value : Number(value); + }); + + if (options.dateGrouping) { + const dateGroupVarName = reverseMapping['dateGroup']; + group.dateGroup = result[dateGroupVarName].value; + } + + const countVarName = reverseMapping['count']; + const entityIdsVarName = reverseMapping['entityIds']; + + return { + group, + count: parseInt(result[countVarName].value, 10), + entityIds: result[entityIdsVarName].value.split(" "), + }; + }); + + return { + results: groupResults, + meta: { + totalCount: groupResults.reduce((sum, curr) => sum + curr.count, 0), + dateRange: options.dateRange, + groupings: options.groupBy || [], + }, + }; + } + public async update(id: string, attributes: Partial): Promise; public async update(ids: string[], attributes: Partial): Promise; public async update(idOrIds: string | string[], attributes: Partial): Promise { diff --git a/src/storage/query-adapter/sparql/SparqlQueryBuilder.ts b/src/storage/query-adapter/sparql/SparqlQueryBuilder.ts index ca50f19..9e7856b 100644 --- a/src/storage/query-adapter/sparql/SparqlQueryBuilder.ts +++ b/src/storage/query-adapter/sparql/SparqlQueryBuilder.ts @@ -44,6 +44,7 @@ import { createSparqlExistsOperation, createSparqlContainsOperation, createSparqlLcaseOperation, + createSparqlSelectQuery, } from '../../../util/SparqlUtil'; import { valueToLiteral, @@ -68,6 +69,7 @@ import type { import type { InverseRelationOperatorValue } from '../../operator/InverseRelation'; import type { InverseRelationOrderValue } from '../../operator/InverseRelationOrder'; import { VariableGenerator } from './VariableGenerator'; +import { GroupByOptions, GroupByResponse } from '../../GroupOptionTypes'; export interface NonGraphWhereQueryData { values: ValuesPattern[]; @@ -997,4 +999,252 @@ export class SparqlQueryBuilder { } return patterns; } + + private createGroupPatternForPath(entityVariable: Variable, path: string): { + variable: Variable; + patterns: Pattern[]; + } { + const segments = path.split('~'); + let currentSubject = entityVariable; + const patterns: Pattern[] = []; + + // Create a chain of patterns for each segment + segments.forEach((predicate, index) => { + const object = this.createVariable(); + patterns.push({ + type: 'bgp', + triples: [{ + subject: currentSubject, + predicate: DataFactory.namedNode(predicate), + object: object + }] + }); + currentSubject = object; + }); + + // Return the final variable (last object) and all patterns + return { + variable: currentSubject, + patterns + }; + } + + public async buildGroupByQuery(options: GroupByOptions): Promise<{ query: SelectQuery; variableMapping: Record }> { + const entityVariable = DataFactory.variable('entity'); + const queryData = this.buildEntitySelectPatternsFromOptions( + entityVariable, + { + where: options.where || {}, + } + ); + + // Add group variables and patterns with mapping + const groupVariables: Variable[] = []; + const groupPatterns: Pattern[] = []; + const variableMapping: Record = {}; + + if (options.groupBy) { + options.groupBy.forEach((path) => { + const { variable: groupVar, patterns } = this.createGroupPatternForPath(entityVariable, path); + groupVariables.push(groupVar); + variableMapping[groupVar.value] = path; + groupPatterns.push(...patterns); + }); + } + + // Add date handling if specified + if (options.dateRange) { + const dateVar = this.createVariable(); + variableMapping[dateVar.value] = 'date'; + + const datePattern:Pattern = { + type: 'bgp', + triples: [{ + subject: entityVariable, + predicate: DataFactory.namedNode('http://purl.org/dc/terms/created'), + object: dateVar + }] + }; + + const dateFilter: FilterPattern = { + type: 'filter', + expression: { + type: 'operation', + operator: '&&', + args: [ + { + type: 'operation', + operator: '>=', + args: [dateVar, DataFactory.literal(options.dateRange.start, 'http://www.w3.org/2001/XMLSchema#dateTime')] + }, + { + type: 'operation', + operator: '<=', + args: [dateVar, DataFactory.literal(options.dateRange.end, 'http://www.w3.org/2001/XMLSchema#dateTime')] + } + ] + } + }; + + groupPatterns.push(datePattern, dateFilter); + + if (options.dateGrouping) { + const dateGroupVar = this.createVariable(); + groupVariables.push(dateGroupVar); + variableMapping[dateGroupVar.value] = 'dateGroup'; + + const dateGroupBind = this.createDateGroupingBind(dateVar, dateGroupVar, options.dateGrouping); + groupPatterns.push(dateGroupBind); + } + } + + // Create count and entityIds variables + const countVar = this.createVariable(); + const entityIdsVar = this.createVariable(); + variableMapping[countVar.value] = 'count'; + variableMapping[entityIdsVar.value] = 'entityIds'; + + // Combine all patterns + const combinedWhere = [ + ...queryData.where, + ...groupPatterns + ]; + + // Create select query with aggregations + const selectQuery = createSparqlSelectQuery( + [ + ...groupVariables, + { + expression: { + type: 'aggregate', + aggregation: 'count', + distinct: true, + expression: entityVariable + }, + variable: countVar + }, + { + expression: { + type: 'aggregate', + aggregation: 'group_concat', + distinct: true, + separator: ' ', + expression: entityVariable + }, + variable: entityIdsVar + } + ], + combinedWhere, + [], // orders + groupVariables, // group by + options.limit, + options.offset + ); + + return { query: selectQuery, variableMapping }; + } + + // Helper function for date grouping + private createDateGroupingBind( + dateVar: Variable, + dateGroupVar: Variable, + grouping: 'month' | 'day' + ): Pattern { + return { + type: 'bind', + expression: { + type: 'operation', + operator: 'CONCAT', + args: [ + this.createYearExpression(dateVar), + DataFactory.literal('-'), + this.createMonthExpression(dateVar), + ...this.createDayExpressionParts(dateVar, grouping) + ] + }, + variable: dateGroupVar + }; + } + + private createYearExpression(dateVar: Variable): Expression { + return { + type: 'operation', + operator: 'STR', + args: [{ + type: 'operation', + operator: 'YEAR', + args: [dateVar] + }] + }; + } + + private createMonthExpression(dateVar: Variable): Expression { + return this.createPaddedDatePartExpression(dateVar, 'MONTH'); + } + + private createDayExpression(dateVar: Variable): Expression { + return this.createPaddedDatePartExpression(dateVar, 'DAY'); + } + + private createPaddedDatePartExpression(dateVar: Variable, datePart: 'MONTH' | 'DAY'): Expression { + const comparisonValue = DataFactory.literal('10', DataFactory.namedNode('http://www.w3.org/2001/XMLSchema#integer')); + + return { + type: 'operation', + operator: 'IF', + args: [ + this.createLessThanComparison(dateVar, datePart, comparisonValue), + this.createPaddedStringExpression(dateVar, datePart), + this.createUnpaddedStringExpression(dateVar, datePart) + ] + }; + } + + private createLessThanComparison(dateVar: Variable, datePart: 'MONTH' | 'DAY', comparisonValue: Term): Expression { + return { + type: 'operation', + operator: '<', + args: [ + { + type: 'operation', + operator: datePart, + args: [dateVar] + } as Expression, + comparisonValue as Expression + ] + }; + } + + private createPaddedStringExpression(dateVar: Variable, datePart: 'MONTH' | 'DAY'): Expression { + return { + type: 'operation', + operator: 'CONCAT', + args: [ + DataFactory.literal('0'), + this.createUnpaddedStringExpression(dateVar, datePart) + ] + }; + } + + private createUnpaddedStringExpression(dateVar: Variable, datePart: 'MONTH' | 'DAY'): Expression { + return { + type: 'operation', + operator: 'STR', + args: [{ + type: 'operation', + operator: datePart, + args: [dateVar] + }] + }; + } + + private createDayExpressionParts(dateVar: Variable, grouping: 'month' | 'day'): Expression[] { + if (grouping === 'day') { + return [ + DataFactory.literal('-'), + this.createDayExpression(dateVar) + ]; + } + return [DataFactory.literal('')]; + } } diff --git a/src/util/SparqlUtil.ts b/src/util/SparqlUtil.ts index bbbfe8d..f6d9165 100644 --- a/src/util/SparqlUtil.ts +++ b/src/util/SparqlUtil.ts @@ -218,23 +218,44 @@ export function createSparqlServicePattern(serviceName: string, triples: Triple[ }; } +export function ensureVariable(variableLike: string | Variable): Variable { + if (typeof variableLike === "string" && variableLike.startsWith("?")) { + return DataFactory.variable(variableLike.slice(1)); + } + return variableLike as Variable; +} + +export function ensureGrouping(variableLike: Variable | string): Grouping { + return { + expression: ensureVariable(variableLike) + } as Grouping; +} + export function createSparqlSelectQuery( - variable: Variable, + variable: Variable | Variable[], where: Pattern[], order: Ordering[], - group?: Variable, + group?: Variable | Variable[], limit?: number, offset?: number, ): SelectQuery { + let groupings: Grouping[] | undefined = []; + if (group) { + if (Array.isArray(group)) { + groupings = group.map(g => ensureGrouping(g)); + } else { + groupings = [ensureGrouping(group)]; + } + } return { - type: 'query', - queryType: 'SELECT', - variables: [ variable ], + type: "query", + queryType: "SELECT", + variables: Array.isArray(variable) + ? variable.map(ensureVariable) + : [ensureVariable(variable)], distinct: true, where, - group: group - ? [ { expression: group } as Grouping ] - : undefined, + group: groupings, order: order.length > 0 ? order : undefined, limit, offset,