diff --git a/Sources/FeaturevisorTestRunner/Extensions/UInt64+Time.swift b/Sources/FeaturevisorTestRunner/Extensions/UInt64+Time.swift new file mode 100644 index 0000000..7362753 --- /dev/null +++ b/Sources/FeaturevisorTestRunner/Extensions/UInt64+Time.swift @@ -0,0 +1,14 @@ +import Foundation + +extension UInt64 { + + var milliseconds: Double { + let elapsedTimeInMilliSeconds = Double(self) / 1_000_000.0 + return Double(elapsedTimeInMilliSeconds) + } + + var seconds: Double { + let elapsedTimeInMilliSeconds = Double(self) / 1_000_000_000.0 + return Double(elapsedTimeInMilliSeconds) + } +} diff --git a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Print.swift b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Print.swift deleted file mode 100644 index 148782a..0000000 --- a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Print.swift +++ /dev/null @@ -1,51 +0,0 @@ -import FeaturevisorTypes -import Foundation - -extension FeaturevisorTestRunner { - - enum ResultMark: String { - case success = "✔" - case failure = "✘" - - init(result: Bool) { - self = result ? .success : .failure - } - - var mark: String { - return rawValue - } - } - - func printAssertionResult( - environment: Environment, - index: Int, - feature: String, - assertionResult: Bool, - expectedValueFailures: [VariableKey: (expected: VariableValue, got: VariableValue?)], - description: String, - onlyFailures: Bool - ) { - - guard onlyFailures && !assertionResult || !onlyFailures else { - return - } - - print("\nTesting: \(feature).feature.yml") - print(" feature \"\(feature)\"") - - let resultMark = ResultMark(result: assertionResult) - - print(" \(resultMark.mark) Assertion #\(index): \(environment.rawValue) \(description)") - - expectedValueFailures.forEach({ (key, value) in - print(" => variable key: \(key)") - print(" => expected: \(value.expected)") - if let got = value.got { - print(" => received: \(got)") - } - else { - print(" => received: nil") - } - }) - } -} diff --git a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift index 1586b6f..beb9791 100644 --- a/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift +++ b/Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift @@ -32,6 +32,8 @@ struct FeaturevisorTestRunner: ParsableCommand { let features = try loadAllFeatures(featuresTestDirectoryPath: featuresTestDirectoryPath) + var totalElapsedDurationInMilliseconds: UInt64 = 0 + var totalTestSpecs = 0 var failedTestSpecs = 0 @@ -46,6 +48,11 @@ struct FeaturevisorTestRunner: ParsableCommand { return } + let output = FeatureResultOutputBuilder( + feature: testSuit.feature, + onlyFailures: onlyFailures + ) + totalTestSpecs += 1 totalAssertionsCount += testSuit.assertions.count @@ -63,6 +70,7 @@ struct FeaturevisorTestRunner: ParsableCommand { using: featuresTestDirectoryPath, assertionAt: testCase.at ) + let sdkStaging = try SDKProvider.provide( for: tag, under: .staging, @@ -102,6 +110,8 @@ struct FeaturevisorTestRunner: ParsableCommand { in: features ) + let startTime = DispatchTime.now() + isFeatureEnabledResult = sdks[tag]![.staging]! .isEnabled( @@ -125,6 +135,8 @@ struct FeaturevisorTestRunner: ParsableCommand { } }) + let endTime = DispatchTime.now() + let finalAssertionResult = isFeatureEnabledResult && expectedValueFailures.isEmpty @@ -132,14 +144,16 @@ struct FeaturevisorTestRunner: ParsableCommand { failedAssertionsCount = finalAssertionResult ? failedAssertionsCount : failedAssertionsCount + 1 - printAssertionResult( + let elapsedTime = endTime.uptimeNanoseconds - startTime.uptimeNanoseconds + totalElapsedDurationInMilliseconds += elapsedTime + + output.addAssertion( environment: testCase.environment, index: index, - feature: testSuit.feature, assertionResult: finalAssertionResult, expectedValueFailures: expectedValueFailures, description: testCase.description, - onlyFailures: onlyFailures + elapsedTime: elapsedTime ) case .production: @@ -162,6 +176,8 @@ struct FeaturevisorTestRunner: ParsableCommand { in: features ) + let startTime = DispatchTime.now() + isFeatureEnabledResult = sdks[tag]![.production]! .isEnabled( @@ -185,6 +201,8 @@ struct FeaturevisorTestRunner: ParsableCommand { } }) + let endTime = DispatchTime.now() + let finalAssertionResult = isFeatureEnabledResult && expectedValueFailures.isEmpty @@ -192,18 +210,24 @@ struct FeaturevisorTestRunner: ParsableCommand { failedAssertionsCount = finalAssertionResult ? failedAssertionsCount : failedAssertionsCount + 1 - printAssertionResult( + let elapsedTime = endTime.uptimeNanoseconds - startTime.uptimeNanoseconds + totalElapsedDurationInMilliseconds += elapsedTime + + output.addAssertion( environment: testCase.environment, index: index, - feature: testSuit.feature, assertionResult: finalAssertionResult, expectedValueFailures: expectedValueFailures, description: testCase.description, - onlyFailures: onlyFailures + elapsedTime: elapsedTime ) } } + if let output = output.build() { + print(output) + } + failedTestSpecs = isTestSpecFailing ? failedTestSpecs + 1 : failedTestSpecs }) @@ -211,6 +235,8 @@ struct FeaturevisorTestRunner: ParsableCommand { print( "Assertions: \(totalAssertionsCount - failedAssertionsCount) passed, \(failedAssertionsCount) failed" ) + + print("Assertions execution duration: \(totalElapsedDurationInMilliseconds.milliseconds)ms") } } diff --git a/Sources/FeaturevisorTestRunner/Utils/FeatureResultOutputBuilder.swift b/Sources/FeaturevisorTestRunner/Utils/FeatureResultOutputBuilder.swift new file mode 100644 index 0000000..0ace104 --- /dev/null +++ b/Sources/FeaturevisorTestRunner/Utils/FeatureResultOutputBuilder.swift @@ -0,0 +1,99 @@ +import FeaturevisorTypes +import Foundation + +class FeatureResultOutputBuilder { + + enum ResultMark: String { + case success = "✔" + case failure = "✘" + + init(result: Bool) { + self = result ? .success : .failure + } + } + + struct Assertion { + let environment: Environment + let index: Int + let assertionResult: Bool + let expectedValueFailures: [VariableKey: (expected: VariableValue, got: VariableValue?)] + let description: String + let elapsedTime: UInt64 + } + + private let onlyFailures: Bool + private let feature: String + private var assertions: [Assertion] = [] + + init(feature: String, onlyFailures: Bool) { + self.onlyFailures = onlyFailures + self.feature = feature + } + + @discardableResult func addAssertion( + environment: Environment, + index: Int, + assertionResult: Bool, + expectedValueFailures: [VariableKey: (expected: VariableValue, got: VariableValue?)], + description: String, + elapsedTime: UInt64 + ) -> Self { + let assertion: Assertion = .init( + environment: environment, + index: index, + assertionResult: assertionResult, + expectedValueFailures: expectedValueFailures, + description: description, + elapsedTime: elapsedTime + ) + + assertions.append(assertion) + return self + } + + func build() -> String? { + + let hasFailedAssertions = !assertions.filter({ !$0.assertionResult }).isEmpty + + guard onlyFailures && hasFailedAssertions || !onlyFailures else { + return nil + } + + let totalElapsedTimeInMilliSeconds = assertions.reduce( + 0.0, + { $0 + $1.elapsedTime.milliseconds } + ) + var output: String = "" + + output.append("\nTesting: \(feature).feature.yml (\(totalElapsedTimeInMilliSeconds)ms)") + output.append("\n feature \"\(feature)\"") + + assertions.forEach({ assertion in + + let mark = ResultMark(result: assertion.assertionResult) + let index = assertion.index + let env = assertion.environment.rawValue + let description = assertion.description + let expectedValueFailures = assertion.expectedValueFailures + let elapsedTimeInMilliSeconds = assertion.elapsedTime.milliseconds + + output.append( + "\n \(mark.rawValue) Assertion #\(index): \(env) \(description) (\(elapsedTimeInMilliSeconds)ms)" + ) + + expectedValueFailures.forEach({ (key, value) in + output.append("\n => variable key: \(key)") + output.append("\n => expected: \(value.expected)") + + if let got = value.got { + output.append("\n => received: \(got)") + } + else { + output.append("\n => received: nil") + } + }) + }) + + return output + } +}