From 6ccb33e9297eafeb29ab3d2c546776ff52ca311e Mon Sep 17 00:00:00 2001 From: Marcin Polak Date: Fri, 19 Apr 2024 22:53:20 +0200 Subject: [PATCH 1/2] feat: evaluate features in CLI --- README.md | 13 ++ .../FeaturevisorSDK/Instance+Evaluation.swift | 95 ++++++++++----- .../FeaturevisorSDK/Instance+Feature.swift | 28 +++-- Sources/FeaturevisorSDK/Instance.swift | 19 +++ Sources/FeaturevisorSDK/Logger.swift | 2 +- .../Evaluation+CustomStringConvertible.swift | 55 +++++++++ .../FeaturevisorTestRunner+Evaluate.swift | 112 ++++++++++++++++++ .../FeaturevisorTestRunner.swift | 59 ++++++++- .../Models/SDKProvider.swift | 2 + Sources/FeaturevisorTypes/Types.swift | 12 +- 10 files changed, 356 insertions(+), 41 deletions(-) create mode 100644 Sources/FeaturevisorTestRunner/Extensions/Evaluation+CustomStringConvertible.swift create mode 100644 Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Evaluate.swift diff --git a/README.md b/README.md index a646548..939a362 100644 --- a/README.md +++ b/README.md @@ -386,6 +386,19 @@ To benchmark evaluating a feature's variable via SDKs's `.getVariable()` method: -n 100 ``` +### Evaluate +To learn why certain values (like feature and its variation or variables) are evaluated as they are against provided [context](https://featurevisor.com/docs/sdks/javascript/#context): + +```bash + FeaturevisorTestRunner evaluate \ + --environment staging \ + --feature feature_key \ + --feature variable_key \ + --context '{"user_id":"123"}' \ +``` +This will show you full [evaluation details](https://featurevisor.com/docs/sdks/javascript/#evaluation-details) helping you debug better in case of any confusion. +It is similar to logging in SDKs with debug level. But here instead, we are doing it at CLI directly in our Featurevisor project without having to involve our application(s). + ## License [MIT](./LICENSE) diff --git a/Sources/FeaturevisorSDK/Instance+Evaluation.swift b/Sources/FeaturevisorSDK/Instance+Evaluation.swift index a0f759c..f1ef584 100644 --- a/Sources/FeaturevisorSDK/Instance+Evaluation.swift +++ b/Sources/FeaturevisorSDK/Instance+Evaluation.swift @@ -79,11 +79,13 @@ extension FeaturevisorInstance { let finalContext = interceptContext != nil ? interceptContext!(context) : context // forced - if let force = findForceFromFeature( + let forceResult = findForceFromFeature( feature, context: context, datafileReader: datafileReader - ) { + ) + + if let force = forceResult.force { let variation = feature.variations.first(where: { variation in return variation.value == force.variation }) @@ -102,12 +104,12 @@ extension FeaturevisorInstance { } // bucketing - let bucketValue = getBucketValue(feature: feature, context: finalContext) + let bucketResult = getBucketValue(feature: feature, context: finalContext) let matchedTrafficAndAllocation = getMatchedTrafficAndAllocation( traffic: feature.traffic, context: finalContext, - bucketValue: bucketValue, + bucketValue: bucketResult.bucketValue, datafileReader: datafileReader, logger: logger ) @@ -125,7 +127,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .rule, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, ruleKey: matchedTraffic.key, variation: variation ) @@ -147,7 +150,10 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .allocated, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, + ruleKey: matchedTraffic.key, + traffic: matchedTraffic, variation: variation ) @@ -162,7 +168,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .error, - bucketValue: bucketValue + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue ) logger.debug("no matched variation", evaluation.toDictionary()) @@ -225,13 +232,19 @@ extension FeaturevisorInstance { let finalContext = interceptContext != nil ? interceptContext!(context) : context // forced - let force = findForceFromFeature(feature, context: context, datafileReader: datafileReader) + let forceResult = findForceFromFeature( + feature, + context: context, + datafileReader: datafileReader + ) - if let force, force.enabled != nil { + if let force = forceResult.force, force.enabled != nil { evaluation = Evaluation( featureKey: featureKey, reason: .forced, - enabled: force.enabled + enabled: force.enabled, + forceIndex: forceResult.forceIndex, + force: force ) logger.debug("forced enabled found", evaluation.toDictionary()) @@ -284,7 +297,7 @@ extension FeaturevisorInstance { } // bucketing - let bucketValue = getBucketValue(feature: feature, context: finalContext) + let bucketResult = getBucketValue(feature: feature, context: finalContext) let matchedTraffic = getMatchedTraffic( traffic: feature.traffic, @@ -297,7 +310,8 @@ extension FeaturevisorInstance { if !feature.ranges.isEmpty { let matchedRange = feature.ranges.first(where: { range in - return bucketValue >= range.start && bucketValue < range.end + return bucketResult.bucketValue >= range.start + && bucketResult.bucketValue < range.end }) // matched @@ -305,7 +319,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .allocated, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, enabled: matchedTraffic.enabled ?? true ) @@ -316,7 +331,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .outOfRange, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, enabled: false ) @@ -330,7 +346,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .override, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, ruleKey: matchedTraffic.key, enabled: matchedTrafficEnabled, traffic: matchedTraffic @@ -342,11 +359,12 @@ extension FeaturevisorInstance { } // treated as enabled because of matched traffic - if bucketValue <= matchedTraffic.percentage { + if bucketResult.bucketValue <= matchedTraffic.percentage { evaluation = Evaluation( featureKey: feature.key, reason: .rule, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, ruleKey: matchedTraffic.key, enabled: true, traffic: matchedTraffic @@ -360,7 +378,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .error, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, enabled: false ) @@ -451,12 +470,18 @@ extension FeaturevisorInstance { let finalContext = interceptContext != nil ? interceptContext!(context) : context // forced - let force = findForceFromFeature(feature, context: context, datafileReader: datafileReader) + let forceResult = findForceFromFeature( + feature, + context: context, + datafileReader: datafileReader + ) - if let force, let variableValue = force.variables?[variableKey] { + if let force = forceResult.force, let variableValue = force.variables?[variableKey] { evaluation = Evaluation( featureKey: feature.key, reason: .forced, + forceIndex: forceResult.forceIndex, + force: force, variableKey: variableKey, variableValue: variableValue, variableSchema: variableSchema @@ -468,12 +493,12 @@ extension FeaturevisorInstance { } // bucketing - let bucketValue = getBucketValue(feature: feature, context: finalContext) + let bucketResult = getBucketValue(feature: feature, context: finalContext) let matchedTrafficAndAllocation = getMatchedTrafficAndAllocation( traffic: feature.traffic, context: finalContext, - bucketValue: bucketValue, + bucketValue: bucketResult.bucketValue, datafileReader: datafileReader, logger: logger ) @@ -484,8 +509,10 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .rule, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, ruleKey: matchedTraffic.key, + traffic: matchedTraffic, variableKey: variableKey, variableValue: variableValue, variableSchema: variableSchema @@ -499,7 +526,7 @@ extension FeaturevisorInstance { // regular allocation var variationValue: VariationValue? = nil - if let forceVariation = force?.variation { + if let forceVariation = forceResult.force?.variation { variationValue = forceVariation } else if let matchedAllocationVariation = matchedTrafficAndAllocation.matchedAllocation? @@ -543,8 +570,10 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .override, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, ruleKey: matchedTraffic.key, + traffic: matchedTraffic, variableKey: variableKey, variableValue: override.value, variableSchema: variableSchema @@ -560,8 +589,10 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .allocated, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, ruleKey: matchedTraffic.key, + traffic: matchedTraffic, variableKey: variableKey, variableValue: variableFromVariationValue, variableSchema: variableSchema @@ -579,7 +610,8 @@ extension FeaturevisorInstance { evaluation = Evaluation( featureKey: feature.key, reason: .defaulted, - bucketValue: bucketValue, + bucketKey: bucketResult.bucketKey, + bucketValue: bucketResult.bucketValue, variableKey: variableKey, variableValue: variableSchema.defaultValue, variableSchema: variableSchema @@ -645,15 +677,18 @@ extension FeaturevisorInstance { return result } - fileprivate func getBucketValue(feature: Feature, context: Context) -> BucketValue { + public typealias BucketResult = (bucketKey: BucketKey, bucketValue: BucketValue) + + fileprivate func getBucketValue(feature: Feature, context: Context) -> BucketResult { let bucketKey = getBucketKey(feature: feature, context: context) let value = Bucket.resolveNumber(forKey: bucketKey) if let configureBucketValue = self.configureBucketValue { - return configureBucketValue(feature, context, value) + let configuredValue = configureBucketValue(feature, context, value) + return (bucketKey: bucketKey, bucketValue: configuredValue) } - return value + return (bucketKey: bucketKey, bucketValue: value) } } diff --git a/Sources/FeaturevisorSDK/Instance+Feature.swift b/Sources/FeaturevisorSDK/Instance+Feature.swift index 81e008a..8cbff67 100644 --- a/Sources/FeaturevisorSDK/Instance+Feature.swift +++ b/Sources/FeaturevisorSDK/Instance+Feature.swift @@ -13,23 +13,35 @@ extension FeaturevisorInstance { _ feature: Feature, context: Context, datafileReader: DatafileReader - ) -> Force? { + ) -> ForceResult { - return feature.force.first(where: { force in - if let conditions = force.conditions { - return allConditionsAreMatched(condition: conditions, context: context) + var force: Force? + var forceIndex: Int? + + for (index, currentForce) in feature.force.enumerated() { + if let condition = currentForce.conditions, + allConditionsAreMatched(condition: condition, context: context) + { + + force = currentForce + forceIndex = index + break } - if let segments = force.segments { - return allGroupSegmentsAreMatched( + if let segments = currentForce.segments, + allGroupSegmentsAreMatched( groupSegments: segments, context: context, datafileReader: datafileReader ) + { + force = currentForce + forceIndex = index + break } + } - return false - }) + return .init(force: force, forceIndex: forceIndex) } func getMatchedTraffic( diff --git a/Sources/FeaturevisorSDK/Instance.swift b/Sources/FeaturevisorSDK/Instance.swift index 4518dea..a590546 100644 --- a/Sources/FeaturevisorSDK/Instance.swift +++ b/Sources/FeaturevisorSDK/Instance.swift @@ -32,9 +32,13 @@ public struct Evaluation: Codable { private enum CodingKeys: String, CodingKey { case featureKey case reason + case bucketKey case bucketValue case ruleKey + case error case enabled + case forceIndex + case force case traffic case sticky case initial @@ -50,10 +54,13 @@ public struct Evaluation: Codable { public let reason: EvaluationReason // common + public let bucketKey: BucketKey? public let bucketValue: BucketValue? public let ruleKey: RuleKey? public let enabled: Bool? public let traffic: Traffic? + public let forceIndex: Int? + public let force: Force? public let sticky: OverrideFeature? public let initial: OverrideFeature? @@ -69,10 +76,13 @@ public struct Evaluation: Codable { public init( featureKey: FeatureKey, reason: EvaluationReason, + bucketKey: BucketKey? = nil, bucketValue: BucketValue? = nil, ruleKey: RuleKey? = nil, enabled: Bool? = nil, traffic: Traffic? = nil, + forceIndex: Int? = nil, + force: Force? = nil, sticky: OverrideFeature? = nil, initial: OverrideFeature? = nil, variation: Variation? = nil, @@ -83,10 +93,13 @@ public struct Evaluation: Codable { ) { self.featureKey = featureKey self.reason = reason + self.bucketKey = bucketKey self.bucketValue = bucketValue self.ruleKey = ruleKey self.enabled = enabled self.traffic = traffic + self.forceIndex = forceIndex + self.force = force self.sticky = sticky self.initial = initial self.variation = variation @@ -101,9 +114,12 @@ public struct Evaluation: Codable { try container.encode(featureKey, forKey: .featureKey) try container.encode(reason.rawValue, forKey: .reason) + try container.encodeIfPresent(bucketKey, forKey: .bucketKey) try container.encodeIfPresent(bucketValue, forKey: .bucketValue) try container.encodeIfPresent(ruleKey, forKey: .ruleKey) try container.encodeIfPresent(enabled, forKey: .enabled) + try container.encodeIfPresent(forceIndex, forKey: .forceIndex) + try container.encodeIfPresent(force, forKey: .force) try container.encodeIfPresent(traffic, forKey: .traffic) try container.encodeIfPresent(sticky, forKey: .sticky) try container.encodeIfPresent(initial, forKey: .initial) @@ -120,9 +136,12 @@ public struct Evaluation: Codable { featureKey = try container.decode(FeatureKey.self, forKey: .featureKey) reason = try EvaluationReason(rawValue: container.decode(String.self, forKey: .reason)) ?? .error + bucketKey = try container.decodeIfPresent(BucketKey.self, forKey: .bucketKey) bucketValue = try container.decodeIfPresent(BucketValue.self, forKey: .bucketValue) ruleKey = try? container.decodeIfPresent(RuleKey.self, forKey: .ruleKey) enabled = try? container.decodeIfPresent(Bool.self, forKey: .enabled) + forceIndex = try? container.decodeIfPresent(Int.self, forKey: .forceIndex) + force = try? container.decodeIfPresent(Force.self, forKey: .force) traffic = try? container.decodeIfPresent(Traffic.self, forKey: .traffic) sticky = try? container.decodeIfPresent(OverrideFeature.self, forKey: .sticky) initial = try? container.decodeIfPresent(OverrideFeature.self, forKey: .initial) diff --git a/Sources/FeaturevisorSDK/Logger.swift b/Sources/FeaturevisorSDK/Logger.swift index aa3c3f7..55eaed6 100644 --- a/Sources/FeaturevisorSDK/Logger.swift +++ b/Sources/FeaturevisorSDK/Logger.swift @@ -24,7 +24,7 @@ public class Logger { var levels: [LogLevel] let handle: LogHandler - init(levels: [LogLevel], handle: @escaping LogHandler) { + public init(levels: [LogLevel], handle: @escaping LogHandler) { self.levels = levels self.handle = handle } diff --git a/Sources/FeaturevisorTestRunner/Extensions/Evaluation+CustomStringConvertible.swift b/Sources/FeaturevisorTestRunner/Extensions/Evaluation+CustomStringConvertible.swift new file mode 100644 index 0000000..b7bf7d2 --- /dev/null +++ b/Sources/FeaturevisorTestRunner/Extensions/Evaluation+CustomStringConvertible.swift @@ -0,0 +1,55 @@ +import FeaturevisorSDK + +extension Evaluation: EvaluationStringConvertible {} + +private protocol EvaluationStringConvertible: CustomStringConvertible {} + +extension EvaluationStringConvertible { + + public var description: String { + let ignoreKeys = [ + "featureKey", "variableKey", "variation", "variableSchema", "traffic", "force", + ] + + let mirror = Mirror(reflecting: self) + + let output: [String] = mirror + .allChildren + .sorted { + $0.label ?? "" < $1.label ?? "" + } + .compactMap { (key: String?, value: Any) in + guard let key, !ignoreKeys.contains(key) else { + return nil + } + + switch value { + case Optional.none: + return nil + case Optional.some(let wrapped): + return "- \(key): \(wrapped)" + default: + return nil + } + } + + return "\(output.joined(separator: "\n"))" + } +} + +extension Mirror { + + /// The children of the mirror and its superclasses. + fileprivate var allChildren: [Mirror.Child] { + var children = Array(self.children) + + var superclassMirror = self.superclassMirror + + while let mirror = superclassMirror { + children.append(contentsOf: mirror.children) + superclassMirror = mirror.superclassMirror + } + + return children + } +} diff --git a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Evaluate.swift b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Evaluate.swift new file mode 100644 index 0000000..a5f5b68 --- /dev/null +++ b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Evaluate.swift @@ -0,0 +1,112 @@ +import Commands +import FeaturevisorSDK +import FeaturevisorTypes +import Foundation + +extension FeaturevisorTestRunner.Evaluate { + + func evaluateFeature(options: Options) { + + // TODO: Handle this better + Commands.Task.run("bash -c featurevisor build") + + let f = try! SDKProvider.provide( + for: .ios, + under: options.environment, + using: ".", + assertionAt: 1 + ) + + let flagEvaluation = f.evaluateFlag(featureKey: options.feature, context: options.context) + let variationEvaluation = f.evaluateVariation( + featureKey: options.feature, + context: options.context + ) + + var variableEvaluations: [VariableKey: Evaluation] = [:] + let feature = f.getFeature(byKey: options.feature) + + feature?.variablesSchema + .forEach({ variableSchema in + let variableEvaluation = f.evaluateVariable( + featureKey: options.feature, + variableKey: variableSchema.key, + context: options.context + ) + + variableEvaluations[variableSchema.key] = variableEvaluation + }) + + print( + "Evaluating feature \(options.feature) in environment \(options.environment.rawValue)..." + ) + print("Against context: \(options.context)") // TODO: pretty + + // flag + printHeader("Is enabled?") + + print("Value: \(flagEvaluation.enabled ?? false)") + print("\nDetails:\n") + + printEvaluationDetails(flagEvaluation) + + // variation + printHeader("Variation") + + if let variation = variationEvaluation.variation { + print("Value: \(variation.value)") + + print("\nDetails:\n") + + printEvaluationDetails(variationEvaluation) + } + else { + print("No variations defined.") + } + + // variables + if !variableEvaluations.isEmpty { + variableEvaluations.forEach({ variableKey, evaluation in + printHeader("Variable: \(variableKey)") + + if let variableValue = evaluation.variableValue { + print("Value: \(variableValue)") + } + else { + print("Value: nil") + } + + print("\nDetails:\n") + + printEvaluationDetails(evaluation) + }) + } + else { + printHeader("Variables") + print("No variables defined.") + } + } +} + +extension FeaturevisorTestRunner.Evaluate { + + fileprivate func printHeader(_ message: String) { + print("\n\n###############") + print(" \(message)") + print("###############\n") + } + + fileprivate func printEvaluationDetails(_ evaluation: Evaluation) { + + if let variation = evaluation.variation { + print("- variation: \(variation.value)") + } + + if let variableSchema = evaluation.variableSchema { + print("- variableType: \(variableSchema.type)") + print("- defaultValue: \(variableSchema.defaultValue)") + } + + print(evaluation) + } +} diff --git a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift index 71959d2..b215666 100644 --- a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift +++ b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift @@ -11,7 +11,7 @@ struct FeaturevisorTestRunner: ParsableCommand { static let configuration = CommandConfiguration( abstract: "Featurevisor SDK utilities.", - subcommands: [Test.self, Benchmark.self] + subcommands: [Benchmark.self, Evaluate.self, Test.self] ) } @@ -95,6 +95,63 @@ extension FeaturevisorTestRunner { } } +extension FeaturevisorTestRunner { + + struct Evaluate: ParsableCommand { + + struct Options { + let environment: Environment + let feature: FeatureKey + let context: [AttributeKey: AttributeValue] + } + + struct Output { + let value: Any? + let duration: TimeInterval + } + + static let configuration = CommandConfiguration( + abstract: + "To learn why certain values (like feature and its variation or variables) are evaluated as they are." + ) + + @Option( + help: + "The option is used to specify the environment which will be used for the evaluation run." + ) + var environment: String + + @Option( + help: + "The option is used to specify the feature key which will be used for the evaluation run." + ) + var feature: String + + @Option( + help: + "The option is used to specify the context which will be used for the benchmark run." + ) + var context: String + + mutating func run() throws { + + let _context = try JSONDecoder() + .decode( + [AttributeKey: AttributeValue].self, + from: context.data(using: .utf8)! + ) + + let options: Options = .init( + environment: .init(rawValue: environment)!, + feature: feature, + context: _context + ) + + evaluateFeature(options: options) + } + } +} + extension FeaturevisorTestRunner { struct Test: ParsableCommand { diff --git a/Sources/FeaturevisorTestRunner/Models/SDKProvider.swift b/Sources/FeaturevisorTestRunner/Models/SDKProvider.swift index d611b35..bbfad99 100644 --- a/Sources/FeaturevisorTestRunner/Models/SDKProvider.swift +++ b/Sources/FeaturevisorTestRunner/Models/SDKProvider.swift @@ -46,6 +46,8 @@ enum SDKProvider { options.configureBucketValue = { _, _, _ -> BucketValue in return Int(assertionAt * (maxBucketedNumber / 100.0)) } + options.logger = Logger(levels: []) { _, _, _ in + } return try FeaturevisorSDK.createInstance(options: options) } diff --git a/Sources/FeaturevisorTypes/Types.swift b/Sources/FeaturevisorTypes/Types.swift index 975504a..ec9ae2a 100644 --- a/Sources/FeaturevisorTypes/Types.swift +++ b/Sources/FeaturevisorTypes/Types.swift @@ -505,7 +505,17 @@ public typealias FeatureKey = String public typealias VariableValues = [VariableKey: VariableValue] -public struct Force: Decodable { +public struct ForceResult { + public let force: Force? + public let forceIndex: Int? + + public init(force: Force?, forceIndex: Int?) { + self.force = force + self.forceIndex = forceIndex + } +} + +public struct Force: Codable { public let variation: VariationValue? public let variables: VariableValues? From aea47c17cd4a7d3eea082e0c692801c8712eb61d Mon Sep 17 00:00:00 2001 From: Marcin Polak Date: Fri, 26 Apr 2024 09:19:04 +0200 Subject: [PATCH 2/2] readme update: remove wrong parameter for evaluate command --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 939a362..eaff4c7 100644 --- a/README.md +++ b/README.md @@ -393,7 +393,6 @@ To learn why certain values (like feature and its variation or variables) are ev FeaturevisorTestRunner evaluate \ --environment staging \ --feature feature_key \ - --feature variable_key \ --context '{"user_id":"123"}' \ ``` This will show you full [evaluation details](https://featurevisor.com/docs/sdks/javascript/#evaluation-details) helping you debug better in case of any confusion.