From a1c397110c541a5eb0499a2086c08b5286190ad5 Mon Sep 17 00:00:00 2001 From: brysonjbest <103070659+brysonjbest@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:26:20 -0800 Subject: [PATCH 1/4] Fix pattern of unique inputs to correctly capture expression transformations. --- src/api/ruleMapping/ruleMapping.service.ts | 23 ++++++++++------------ 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/api/ruleMapping/ruleMapping.service.ts b/src/api/ruleMapping/ruleMapping.service.ts index 84334b0..d1eba09 100644 --- a/src/api/ruleMapping/ruleMapping.service.ts +++ b/src/api/ruleMapping/ruleMapping.service.ts @@ -101,20 +101,17 @@ export class RuleMappingService { // inputs that are only transformed are still included as unique as marked as exception async extractUniqueInputs(nodes: Node[]): Promise<{ uniqueInputs: any[] }> { const { inputs, outputs } = await this.extractInputsAndOutputs(nodes); - const outputFields = new Set( - outputs - // check for exceptions where input is transformed and exclude from output fields - .filter((outputField) => - outputField.exception - ? outputField.exception.includes(outputField.key) - ? outputField.exception === outputField.key - : true - : true, - ) - .map((outputField) => outputField.field), - ); - const uniqueInputFields = this.findUniqueFields(inputs, outputFields); + const nonUniqueFields = new Set(outputs.filter((output) => !output.exception).map((output) => output.field)); + + outputs + .filter((output) => output.exception) + .forEach((output) => { + if (output.field === output.key || inputs.some((input) => input.field === output.field)) { + nonUniqueFields.add(output.field); + } + }); + const uniqueInputFields = this.findUniqueFields(inputs, nonUniqueFields); return { uniqueInputs: Object.values(uniqueInputFields), }; From 0c65f7f077d843525a416ada72d7625c75c83d26 Mon Sep 17 00:00:00 2001 From: brysonjbest <103070659+brysonjbest@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:22:28 -0800 Subject: [PATCH 2/4] Add explicit exceptions for improved input handling. --- src/api/ruleMapping/ruleMapping.service.ts | 55 ++++++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/src/api/ruleMapping/ruleMapping.service.ts b/src/api/ruleMapping/ruleMapping.service.ts index d1eba09..4e8e4f6 100644 --- a/src/api/ruleMapping/ruleMapping.service.ts +++ b/src/api/ruleMapping/ruleMapping.service.ts @@ -96,22 +96,57 @@ export class RuleMappingService { return uniqueFields; } + // check for presence of goRules expression in string + private isExpressionFunction(expression: string): boolean { + const functionPatterns = [ + 'none(', + 'map(', + 'flatMap(', + 'filter(', + 'some(', + 'all(', + 'count(', + 'contains(', + 'flatten(', + 'sum(', + 'avg(', + 'min(', + 'max(', + 'mean(', + 'mode(', + 'len(', + '$root', + ]; + return functionPatterns.some((pattern) => expression.includes(pattern)); + } + + // Check if the key is being transformed (used in an operation) + private isTransformation(expression: string, key: string): boolean { + const cleanExpression = expression.replace(/\s/g, ''); + if (cleanExpression === key) return true; + const operatorPattern = new RegExp(`${key}[^=]*[+\\-*/%?:]`); + return operatorPattern.test(expression); + } + // extract only the unique inputs from a list of nodes // excludes inputs found in the outputs of other nodes // inputs that are only transformed are still included as unique as marked as exception async extractUniqueInputs(nodes: Node[]): Promise<{ uniqueInputs: any[] }> { const { inputs, outputs } = await this.extractInputsAndOutputs(nodes); - const nonUniqueFields = new Set(outputs.filter((output) => !output.exception).map((output) => output.field)); - - outputs - .filter((output) => output.exception) - .forEach((output) => { - if (output.field === output.key || inputs.some((input) => input.field === output.field)) { - nonUniqueFields.add(output.field); - } - }); + const outputFields = new Set( + outputs + .filter((outputField) => { + if (!outputField.exception) return true; + if (!outputField.key) return true; + if (this.isExpressionFunction(outputField.exception)) { + return true; + } + return !this.isTransformation(outputField.exception, outputField.key); + }) + .map((outputField) => outputField.field), + ); + const uniqueInputFields = this.findUniqueFields(inputs, outputFields); - const uniqueInputFields = this.findUniqueFields(inputs, nonUniqueFields); return { uniqueInputs: Object.values(uniqueInputFields), }; From 531d6bd9cbbdc3a5e3c61c419a23a389b09ccefb Mon Sep 17 00:00:00 2001 From: brysonjbest <103070659+brysonjbest@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:32:24 -0800 Subject: [PATCH 3/4] Fix handling of date in validations. --- .../validations/validations.service.ts | 33 +++++++++++--- src/api/scenarioData/scenarioData.service.ts | 43 +++++++++++++----- src/utils/csv.spec.ts | 45 ++++++++++++------- src/utils/csv.ts | 33 +++++++++----- 4 files changed, 108 insertions(+), 46 deletions(-) diff --git a/src/api/decisions/validations/validations.service.ts b/src/api/decisions/validations/validations.service.ts index d0c6483..47233c2 100644 --- a/src/api/decisions/validations/validations.service.ts +++ b/src/api/decisions/validations/validations.service.ts @@ -182,20 +182,28 @@ export class ValidationService { private validateDateCriteria(field: any, input: string | string[]): void { const { validationCriteria, validationType } = field; + const parseValue = (value: string) => { + const cleanValue = value + ?.trim() + ?.replace(/[\[\]()]/g, '') + ?.trim(); + return cleanValue?.toLowerCase() === 'today' ? new Date() : new Date(cleanValue); + }; + const dateValues = validationCriteria ? validationCriteria .replace(/[\[\]]/g, '') .split(',') - .map((val: string) => new Date(val.trim()).getTime()) + .map((val: string) => parseValue(val.trim()).getTime()) : []; - const dateValidationValue = new Date(validationCriteria).getTime() || new Date().getTime(); + const dateValidationValue = parseValue(validationCriteria).getTime(); const [minDate, maxDate] = dateValues.length > 1 ? [dateValues[0], dateValues[dateValues.length - 1]] : [new Date().getTime(), new Date().setFullYear(new Date().getFullYear() + 1)]; - const dateInput = typeof input === 'string' ? new Date(input).getTime() : null; + const dateInput = typeof input === 'string' ? parseValue(input).getTime() : null; switch (validationType) { case '==': @@ -269,6 +277,15 @@ export class ValidationService { } } + private normalizeText(text: string): string { + return text + ?.trim() + ?.replace(/[\[\]()]/g, '') + ?.replace(/[''"]/g, '') + ?.replace(/\s+/g, ' ') + ?.trim(); + } + private validateTextCriteria(field: any, input: string | string[]): void { const { validationCriteria, validationType } = field; @@ -310,16 +327,18 @@ export class ValidationService { const validTextArray = validationCriteria .replace(/[\[\]]/g, '') .split(',') - .map((val: string) => val.trim()); + .map((val: string) => this.normalizeText(val)); + const inputArray = Array.isArray(input) - ? input + ? input.map((val) => this.normalizeText(val)) : input .replace(/[\[\]]/g, '') .split(',') - .map((val) => val.trim()); + .map((val) => this.normalizeText(val)); + if (!inputArray.every((inp: string) => validTextArray.includes(inp))) { throw new ValidationError( - `Input ${field.field} must be on or many of the values from: ${validTextArray.join(', ')}`, + `Input ${field.field} must be one or many of the values from: ${validTextArray.join(', ')}`, ); } break; diff --git a/src/api/scenarioData/scenarioData.service.ts b/src/api/scenarioData/scenarioData.service.ts index ef837f4..e9b279b 100644 --- a/src/api/scenarioData/scenarioData.service.ts +++ b/src/api/scenarioData/scenarioData.service.ts @@ -267,9 +267,22 @@ export class ScenarioDataService { return scenarios; } + private cleanValue(value: string): string { + return ( + value + ?.trim() + ?.replace(/[\[\]()]/g, '') + ?.trim() ?? '' + ); + } + generatePossibleValues(input: any, defaultValue?: any): any[] { const { type, dataType, validationCriteria, validationType, childFields } = input; - //Determine how many versions of each field to generate + const parseValue = (value: string) => { + const cleaned = this.cleanValue(value); + return cleaned?.toLowerCase() === 'today' ? new Date() : new Date(cleaned); + }; + const complexityGeneration = 10; if (defaultValue !== null && defaultValue !== undefined) return [defaultValue]; @@ -288,9 +301,9 @@ export class ScenarioDataService { return scenarios; case 'number-input': - const numberValues = validationCriteria?.split(',').map((val: string) => val.trim()); + const numberValues = validationCriteria?.split(',').map((val: string) => this.cleanValue(val)); const minValue = (numberValues && parseInt(numberValues[0], 10)) || 0; - + console.log(numberValues, 'number values'); const maxValue = numberValues && numberValues[numberValues?.length - 1] !== minValue.toString() ? numberValues[numberValues?.length - 1] @@ -323,19 +336,22 @@ export class ScenarioDataService { } case 'date': - const dateValues = validationCriteria?.split(',').map((val: string) => new Date(val.trim()).getTime()); + const dateValues = validationCriteria + ?.split(',') + .map((val: string) => parseValue(this.cleanValue(val)).getTime()); + console.log(dateValues, 'these are date values', validationCriteria); const minDate = (dateValues && dateValues[0]) || new Date().getTime(); const maxDate = dateValues && dateValues[dateValues?.length - 1] !== minDate ? dateValues[dateValues?.length - 1] : new Date().setFullYear(new Date().getFullYear() + 1); + console.log(minDate, maxDate, 'dates'); const generateRandomDates = (count: number) => Array.from({ length: count }, () => new Date(minDate + Math.random() * (maxDate - minDate)).toISOString().slice(0, 10), ); switch (validationType) { case '>=': - return generateRandomDates(complexityGeneration); case '<=': return generateRandomDates(complexityGeneration); case '>': @@ -344,28 +360,31 @@ export class ScenarioDataService { return generateRandomDates(complexityGeneration).filter((date) => new Date(date).getTime() < maxDate); // range exclusive case '(date)': - return generateRandomDates(complexityGeneration).filter( - (date) => new Date(date).getTime() > minDate && new Date(date).getTime() < maxDate, - ); - // range inclusive + return generateRandomDates(complexityGeneration).filter((date) => { + const dateTime = parseValue(date).getTime(); + return dateTime > minDate && dateTime < maxDate; + }); case '[date]': return generateRandomDates(complexityGeneration); case '[=date]': case '[=dates]': - return validationCriteria.split(',').map((val: string) => val.trim()); + return validationCriteria.split(',').map((val: string) => { + const parsedDate = parseValue(val.trim()); + return parsedDate.toISOString().slice(0, 10); + }); default: return generateRandomDates(complexityGeneration); } case 'text-input': if (validationType === '[=texts]') { - const textOptionsArray = validationCriteria.split(',').map((val: string) => val.trim()); + const textOptionsArray = validationCriteria.split(',').map((val: string) => this.cleanValue(val)); const arrayCombinations = generateCombinationsWithLimit(textOptionsArray); return arrayCombinations; } if (validationType === '[=text]') { - return validationCriteria.split(',').map((val: string) => val.trim()); + return validationCriteria.split(',').map((val: string) => this.cleanValue(val)); } // TODO: Future update to include regex generation const generateRandomText = ( diff --git a/src/utils/csv.spec.ts b/src/utils/csv.spec.ts index b662936..83aca22 100644 --- a/src/utils/csv.spec.ts +++ b/src/utils/csv.spec.ts @@ -58,37 +58,52 @@ describe('CSV Utility Functions', () => { }); describe('generateCombinationsWithLimit', () => { - it('should generate all combinations for a small input', () => { + const limit = 10; + it('should generate combinations from input array', () => { const input = ['a', 'b', 'c']; - const result = generateCombinationsWithLimit(input); - const expectedResult = [['a'], ['a', 'b'], ['a', 'b', 'c'], ['a', 'c'], ['b'], ['b', 'c'], ['c']]; - expect(result).toEqual(expectedResult); + const result = generateCombinationsWithLimit(input, limit); + + // Check that results are arrays of strings from input + expect(result.length).toBeGreaterThan(0); + expect( + result.every((combo) => { + return Array.isArray(combo) && combo.length > 0 && combo.every((item) => input.includes(item)); + }), + ).toBe(true); }); it('should respect the limit parameter', () => { const input = ['a', 'b', 'c', 'd', 'e']; - const result = generateCombinationsWithLimit(input, 10); - expect(result.length).toBe(10); + const result = generateCombinationsWithLimit(input, limit); + expect(result.length).toBeLessThanOrEqual(limit); + }); + + it('should generate unique combinations', () => { + const input = ['a', 'b', 'c']; + const result = generateCombinationsWithLimit(input, limit); + + const uniqueCombos = new Set(result.map((combo) => JSON.stringify(combo.sort()))); + expect(uniqueCombos.size).toBe(result.length); }); it('should handle empty input array', () => { const input: string[] = []; - const result = generateCombinationsWithLimit(input); + const result = generateCombinationsWithLimit(input, limit); expect(result).toEqual([]); }); it('should handle single element input array', () => { const input = ['a']; - const result = generateCombinationsWithLimit(input); - expect(result).toEqual([['a']]); + const result = generateCombinationsWithLimit(input, limit); + expect(result.length).toBe(1); + expect(result[0]).toEqual(['a']); }); - it('should handle large input without exceeding memory limits', () => { - const largeInput = Array(20) - .fill(0) - .map((_, i) => String.fromCharCode(97 + i)); - const result = generateCombinationsWithLimit(largeInput, 1000000); - expect(result.length).toBe(1000000); + it('should generate combinations of varying lengths', () => { + const input = ['a', 'b', 'c', 'd']; + const result = generateCombinationsWithLimit(input, limit); + const hasVariableLengths = result.some((combo) => combo.length !== result[0].length); + expect(hasVariableLengths).toBe(true); }); }); diff --git a/src/utils/csv.ts b/src/utils/csv.ts index 0507594..e65bc35 100644 --- a/src/utils/csv.ts +++ b/src/utils/csv.ts @@ -126,23 +126,32 @@ export const complexCartesianProduct = (arrays: T[][], limit: number = 3000): }; /** - * Generates all combinations of a given array with varying lengths. + * Generates random combinations of items from an array with varying lengths. * @param arr The input array to generate combinations from. - * @param limit The maximum number of combinations to generate. - * @returns The generated product. + * @param limit The maximum number of combinations to generate (default: 1000). + * @returns Array of combinations, each containing 1 to n items from the input array. */ export const generateCombinationsWithLimit = (arr: string[], limit: number = 1000): string[][] => { const result: string[][] = []; + if (arr.length == 0) return result; + const getRandomItems = (items: string[], minCount: number = 1): string[] => { + const count = Math.floor(Math.random() * (items.length - minCount + 1)) + minCount; + const shuffled = [...items].sort(() => Math.random() - 0.5); + return shuffled.slice(0, count); + }; + + while (result.length < limit) { + const combination = getRandomItems(arr); - const combine = (prefix: string[], remaining: string[], start: number) => { - if (result.length >= limit) return; // Stop when the limit is reached - for (let i = start; i < remaining.length; i++) { - const newPrefix = [...prefix, remaining[i]]; - result.push(newPrefix); - combine(newPrefix, remaining, i + 1); + // Check for combination uniqueness + const combinationStr = JSON.stringify(combination.sort()); + if (!result.some((existing) => JSON.stringify(existing.sort()) === combinationStr)) { + result.push(combination); } - }; - combine([], arr, 0); - return result.slice(0, limit); + // Break if no more unique combinations + if (result.length === Math.pow(2, arr.length) - 1) break; + } + + return result; }; From 2067a71a460b6c06ea40c5c6cc629f69665bb42e Mon Sep 17 00:00:00 2001 From: brysonjbest <103070659+brysonjbest@users.noreply.github.com> Date: Fri, 20 Dec 2024 14:26:07 -0800 Subject: [PATCH 4/4] Remove redundant logs and trims. --- .../decisions/validations/validations.service.ts | 13 ++++--------- src/api/scenarioData/scenarioData.service.ts | 10 +--------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/api/decisions/validations/validations.service.ts b/src/api/decisions/validations/validations.service.ts index 47233c2..1320644 100644 --- a/src/api/decisions/validations/validations.service.ts +++ b/src/api/decisions/validations/validations.service.ts @@ -183,10 +183,7 @@ export class ValidationService { private validateDateCriteria(field: any, input: string | string[]): void { const { validationCriteria, validationType } = field; const parseValue = (value: string) => { - const cleanValue = value - ?.trim() - ?.replace(/[\[\]()]/g, '') - ?.trim(); + const cleanValue = value?.trim()?.replace(/[\[\]()]/g, ''); return cleanValue?.toLowerCase() === 'today' ? new Date() : new Date(cleanValue); }; @@ -194,7 +191,7 @@ export class ValidationService { ? validationCriteria .replace(/[\[\]]/g, '') .split(',') - .map((val: string) => parseValue(val.trim()).getTime()) + .map((val: string) => parseValue(val).getTime()) : []; const dateValidationValue = parseValue(validationCriteria).getTime(); @@ -280,10 +277,8 @@ export class ValidationService { private normalizeText(text: string): string { return text ?.trim() - ?.replace(/[\[\]()]/g, '') - ?.replace(/[''"]/g, '') - ?.replace(/\s+/g, ' ') - ?.trim(); + ?.replace(/[\[\]()'"''""]/g, '') + ?.replace(/\s+/g, ' '); } private validateTextCriteria(field: any, input: string | string[]): void { diff --git a/src/api/scenarioData/scenarioData.service.ts b/src/api/scenarioData/scenarioData.service.ts index e9b279b..8bfdac8 100644 --- a/src/api/scenarioData/scenarioData.service.ts +++ b/src/api/scenarioData/scenarioData.service.ts @@ -268,12 +268,7 @@ export class ScenarioDataService { } private cleanValue(value: string): string { - return ( - value - ?.trim() - ?.replace(/[\[\]()]/g, '') - ?.trim() ?? '' - ); + return value?.trim()?.replace(/[\[\]()]/g, '') ?? ''; } generatePossibleValues(input: any, defaultValue?: any): any[] { @@ -303,7 +298,6 @@ export class ScenarioDataService { case 'number-input': const numberValues = validationCriteria?.split(',').map((val: string) => this.cleanValue(val)); const minValue = (numberValues && parseInt(numberValues[0], 10)) || 0; - console.log(numberValues, 'number values'); const maxValue = numberValues && numberValues[numberValues?.length - 1] !== minValue.toString() ? numberValues[numberValues?.length - 1] @@ -339,13 +333,11 @@ export class ScenarioDataService { const dateValues = validationCriteria ?.split(',') .map((val: string) => parseValue(this.cleanValue(val)).getTime()); - console.log(dateValues, 'these are date values', validationCriteria); const minDate = (dateValues && dateValues[0]) || new Date().getTime(); const maxDate = dateValues && dateValues[dateValues?.length - 1] !== minDate ? dateValues[dateValues?.length - 1] : new Date().setFullYear(new Date().getFullYear() + 1); - console.log(minDate, maxDate, 'dates'); const generateRandomDates = (count: number) => Array.from({ length: count }, () => new Date(minDate + Math.random() * (maxDate - minDate)).toISOString().slice(0, 10),