From c9c4fcb30dccf2976f5ed6d1296b7a5ac79f3a98 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 13:47:35 +0300 Subject: [PATCH 01/72] bump to 0.6.1 --- examples/herd/examples/asynctest/module.ceylon | 4 ++-- source/herd/asynctest/module.ceylon | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/herd/examples/asynctest/module.ceylon b/examples/herd/examples/asynctest/module.ceylon index 3953982..008f2d2 100644 --- a/examples/herd/examples/asynctest/module.ceylon +++ b/examples/herd/examples/asynctest/module.ceylon @@ -31,7 +31,7 @@ license ( ) by( "Lis" ) native( "jvm" ) -module herd.examples.asynctest "0.6.0" { +module herd.examples.asynctest "0.6.1" { shared import ceylon.collection "1.3.1"; shared import ceylon.http.server "1.3.1"; shared import ceylon.http.client "1.3.1"; @@ -41,6 +41,6 @@ module herd.examples.asynctest "0.6.0" { shared import ceylon.file "1.3.1"; shared import ceylon.promise "1.3.1"; shared import ceylon.test "1.3.1"; - shared import herd.asynctest "0.6.0"; + shared import herd.asynctest "0.6.1"; shared import java.base "8"; } diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index aa5f8b6..7839302 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -403,7 +403,7 @@ license ( ) by( "Lis" ) native( "jvm" ) -module herd.asynctest "0.6.0" { +module herd.asynctest "0.6.1" { import java.base "8"; shared import ceylon.test "1.3.1"; import ceylon.collection "1.3.1"; From fe7767e4840593685d1c90928563f3a33a237f53 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 15:14:37 +0300 Subject: [PATCH 02/72] signal rule as #8 issue --- source/herd/asynctest/module.ceylon | 2 +- .../herd/asynctest/rule/LockAccessRule.ceylon | 1 + source/herd/asynctest/rule/SignalRule.ceylon | 124 ++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 source/herd/asynctest/rule/SignalRule.ceylon diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index 7839302..ef022b8 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -344,7 +344,7 @@ test rule or factory. Example, function `doMyTest` will be interrupted if not completed during 1 second: - timeout( 1K ) test async void doMyTest(...) {...} + timeout(1K) test async void doMyTest(...) {...} ------------------------------------------- diff --git a/source/herd/asynctest/rule/LockAccessRule.ceylon b/source/herd/asynctest/rule/LockAccessRule.ceylon index 502a530..ac7e280 100644 --- a/source/herd/asynctest/rule/LockAccessRule.ceylon +++ b/source/herd/asynctest/rule/LockAccessRule.ceylon @@ -5,6 +5,7 @@ import java.util.concurrent.locks { ReentrantLock } + "Tool for controlling access to a shared resource of `Element` type by multiple threads. The resource value is re-initialized to `initial` _before_ each test. diff --git a/source/herd/asynctest/rule/SignalRule.ceylon b/source/herd/asynctest/rule/SignalRule.ceylon new file mode 100644 index 0000000..cf516ee --- /dev/null +++ b/source/herd/asynctest/rule/SignalRule.ceylon @@ -0,0 +1,124 @@ +import herd.asynctest { + AsyncPrePostContext +} +import java.util.concurrent.locks { + ReentrantLock, + Condition +} +import java.util.concurrent { + TimeUnit +} +import java.util.concurrent.atomic { + AtomicInteger +} + + +"Provides suspend / resume capability. The rule might be applied when some thread + has to await another one completes some calculations. + + Example: + + + class MyTest() { + SignalRule rule = SignalRule(); + + test async void myTest(AsyncTestContext context) { + object thread1 extends Thread() { + shared actual void run() { + rule.await(); // awaits until thread2 signals + ... verify thread 2 work results ... + context.complete(); + } + } + thread1.start(); + + object thread2 extends Thread() { + shared actual void run() { + ... do some work ... + rule.signal(); // wakes up thread1 + } + } + thread2.start(); + } + } + " +tagged( "TestRule" ) since( "0.6.1" ) by( "Lis" ) +shared class SignalRule() satisfies TestRule { + + variable ReentrantLock lock = ReentrantLock(); + variable Condition condition = lock.newCondition(); + + // counting of await, signal and signal all operations + variable AtomicInteger awaitCounts = AtomicInteger( 0 ); + variable AtomicInteger signalCounts = AtomicInteger( 0 ); + variable AtomicInteger signalAllCounts = AtomicInteger( 0 ); + + + "Number of times the `await` has been called. + Rested to zero _before each_ test." + shared Integer numberOfAwaits => awaitCounts.get(); + "Number of times the `signal` has been called. + Rested to zero _before each_ test." + shared Integer numberOfSignal => signalCounts.get(); + "Number of times the `signalAll` has been called. + Rested to zero _before each_ test." + shared Integer numberOfSignalAll => signalAllCounts.get(); + + + shared actual void after( AsyncPrePostContext context ) { + context.proceed(); + } + + shared actual void before( AsyncPrePostContext context ) { + awaitCounts = AtomicInteger( 0 ); + signalCounts = AtomicInteger( 0 ); + signalAllCounts = AtomicInteger( 0 ); + lock = ReentrantLock(); + condition = lock.newCondition(); + context.proceed(); + } + + + "Causes the current thread to wait until it is signalled or interrupted, or the specified waiting time elapses. + If `timeMilliseconds` is less or equal to zero then unlimited wait time is applied. + Returns `false` if wait time is elapsed before `signal` or `signalAll` has been called + otherwise returns `true`. + Note: always returns `true` if `timeMilliseconds` is less or equal to zero. " + shared Boolean await( "The maximum time to wait in milliseconds." Integer timeMilliseconds = -1 ) { + lock.lock(); + try { + awaitCounts.incrementAndGet(); + if ( timeMilliseconds > 0 ) { + return condition.await( timeMilliseconds, TimeUnit.milliseconds ); + } + else { + condition.await(); + return true; + } + } + finally { + lock.unlock(); + } + } + + "Wakes up a one awaiter." + shared void signal() { + lock.lock(); + try { + signalCounts.incrementAndGet(); + condition.signal(); + } + finally { lock.unlock(); } + } + + "Wakes up all awaiters." + shared void signalAll() { + lock.lock(); + try { + signalAllCounts.incrementAndGet(); + condition.signalAll(); + } + finally { lock.unlock(); } + } +} + From 23a70d4d70525198232596656ecbb20e3087f0fc Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 16:06:08 +0300 Subject: [PATCH 03/72] rearrange context await/signal and lock --- source/herd/asynctest/ContextBase.ceylon | 5 ++-- source/herd/asynctest/GuardTester.ceylon | 19 +++------------ source/herd/asynctest/Tester.ceylon | 24 ++++++++++++++++--- .../rule/ChainedPrePostContext.ceylon | 7 +++--- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/source/herd/asynctest/ContextBase.ceylon b/source/herd/asynctest/ContextBase.ceylon index 799aef2..70ae48d 100644 --- a/source/herd/asynctest/ContextBase.ceylon +++ b/source/herd/asynctest/ContextBase.ceylon @@ -28,10 +28,9 @@ class ContextBase() { "Await signaling." shared void await() { - if ( running.get() ) { - locker.lock(); + if ( running.get() && locker.tryLock() ) { try { condition.await(); } finally { locker.unlock(); } } - } + } } diff --git a/source/herd/asynctest/GuardTester.ceylon b/source/herd/asynctest/GuardTester.ceylon index b8aaa6f..d6a890c 100644 --- a/source/herd/asynctest/GuardTester.ceylon +++ b/source/herd/asynctest/GuardTester.ceylon @@ -4,9 +4,6 @@ import herd.asynctest.internal { import java.lang { Thread } -import java.util.concurrent.locks { - ReentrantLock -} "The most lower level of the runners - invokes test function itself. @@ -27,10 +24,6 @@ class GuardTester ( extends ContextBase() satisfies AsyncTestContext { - "Callbacks locking." - ReentrantLock locker = ReentrantLock(); - - "Fails and completes the test with uncaught exception from some execution thread." void failWithUncaughtException( Thread t, Throwable e ) { fail( e, "uncaught exception in child thread." ); @@ -55,26 +48,20 @@ class GuardTester ( shared actual void complete( String title ) { if ( running.compareAndSet( true, false ) ) { - locker.lock(); try { context.complete( title ); } - finally { locker.unlock(); } - signal(); + finally { signal(); } } } shared actual void fail( Throwable|Anything() exceptionSource, String title ) { if ( running.get() ) { - locker.lock(); - try { context.fail( exceptionSource, title ); } - finally { locker.unlock(); } + context.fail( exceptionSource, title ); } } shared actual void succeed( String message ) { if ( running.get() ) { - locker.lock(); - try { context.succeed( message ); } - finally { locker.unlock(); } + context.succeed( message ); } } diff --git a/source/herd/asynctest/Tester.ceylon b/source/herd/asynctest/Tester.ceylon index bb0ddd3..bc90123 100644 --- a/source/herd/asynctest/Tester.ceylon +++ b/source/herd/asynctest/Tester.ceylon @@ -20,6 +20,10 @@ import herd.asynctest.runner { AsyncTestRunner } +import java.util.concurrent.locks { + + ReentrantLock +} "* Performs a one test execution (test function + statements). @@ -35,6 +39,10 @@ class Tester ( ) satisfies AsyncMessageContext { + + "Synchronizes output writing." + ReentrantLock locker = ReentrantLock(); + "storage for reports" ArrayList outputs = ArrayList(); @@ -48,7 +56,11 @@ class Tester ( shared actual void complete( String title ) { if ( outputs.empty ) { - if ( title.empty ) { totalState = TestState.success; } + if ( title.empty ) { + locker.lock(); + try { totalState = TestState.success; } + finally { locker.unlock(); } + } else { addOutput( TestState.success, null, title ); } } } @@ -70,8 +82,14 @@ class Tester ( "Adds new output to `outputs`" void addOutput( TestState state, Throwable? error, String title ) { - outputs.add( TestOutput( state, error, ( system.nanoseconds - startTime ) / 1000000, title ) ); - if ( totalState < state ) { totalState = state; } + locker.lock(); + try { + outputs.add( TestOutput( state, error, ( system.nanoseconds - startTime ) / 1000000, title ) ); + if ( totalState < state ) { totalState = state; } + } + finally { + locker.unlock(); + } } "Fails the test with either `Exception` or `AssertionError`." diff --git a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon index 3a010cd..a9de8b1 100644 --- a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon +++ b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon @@ -65,9 +65,10 @@ class ChainedPrePostContext( AsyncPrePostContext context, Iterator Date: Mon, 28 Nov 2016 17:48:29 +0300 Subject: [PATCH 04/72] doc --- source/herd/asynctest/runner/RepeatRunner.ceylon | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/herd/asynctest/runner/RepeatRunner.ceylon b/source/herd/asynctest/runner/RepeatRunner.ceylon index a7f7d9c..4fea8f8 100644 --- a/source/herd/asynctest/runner/RepeatRunner.ceylon +++ b/source/herd/asynctest/runner/RepeatRunner.ceylon @@ -18,6 +18,9 @@ import herd.asynctest { The runner repeats only test function execution! All `before`, `after` and `testRule` callbacks are executed _once_. In order to repeat overal test execution cycle see [[retry]]. + + > Thread-safe. + " see( `function retry` ) tagged( "Runner", "Repeat" ) From c20c05d905d793ff074c465184912b94e315b8ec Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 19:13:57 +0300 Subject: [PATCH 05/72] aborts test with multiple reasons --- source/herd/asynctest/exceptions.ceylon | 32 +++++++++++++++++++ .../rule/ChainedPrePostContext.ceylon | 10 +++--- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/source/herd/asynctest/exceptions.ceylon b/source/herd/asynctest/exceptions.ceylon index 3856402..a3c365f 100644 --- a/source/herd/asynctest/exceptions.ceylon +++ b/source/herd/asynctest/exceptions.ceylon @@ -1,3 +1,9 @@ +import ceylon.language.meta { + type +} +import ceylon.test.engine { + TestAbortedException +} "Exception which shows that time out has been reached." since( "0.6.0" ) by( "Lis" ) @@ -12,3 +18,29 @@ since( "0.6.0" ) by( "Lis" ) shared class FactoryReturnsNothing( String factoryTitle ) extends Exception( "Factory ``factoryTitle`` has neither instantiated object or throwed." ) {} + + +"Collects multiple abort reasons in a one exception." +since( "0.6.1" ) by( "Lis" ) +shared class MultipleAbortException ( + "The collected exceptions." shared Throwable[] abortReasons, + "A brief abort description" shared String description = "multiple abort reasons (``abortReasons.size``):" +) + extends TestAbortedException() +{ + + shared actual String message { + value message = StringBuilder(); + message.append( description ); + for ( e in abortReasons ) { + message.appendNewline(); + message.append(" "); + message.append( type( e ).declaration.qualifiedName ); + message.append( "(" ); + message.append( e.message ); + message.append( ")" ); + } + return message.string; + } + +} diff --git a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon index a9de8b1..256e0b3 100644 --- a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon +++ b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon @@ -1,13 +1,11 @@ import herd.asynctest { AsyncPrePostContext, - TestInfo + TestInfo, + MultipleAbortException } import ceylon.collection { ArrayList } -import ceylon.test.engine { - MultipleFailureException -} import java.util.concurrent.atomic { AtomicBoolean } @@ -83,8 +81,8 @@ class ChainedPrePostContext( AsyncPrePostContext context, Iterator Date: Mon, 28 Nov 2016 20:47:01 +0300 Subject: [PATCH 06/72] special case for abort and skip exceptions --- .../asynctest/VariantResultBuilder.ceylon | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/source/herd/asynctest/VariantResultBuilder.ceylon b/source/herd/asynctest/VariantResultBuilder.ceylon index 308a044..a886459 100644 --- a/source/herd/asynctest/VariantResultBuilder.ceylon +++ b/source/herd/asynctest/VariantResultBuilder.ceylon @@ -4,6 +4,10 @@ import ceylon.test { import ceylon.collection { ArrayList } +import ceylon.test.engine { + TestSkippedException, + TestAbortedException +} "Builds test variant result from several test outputs." @@ -31,19 +35,31 @@ shared class VariantResultBuilder() { "Adds test failure report to the test results." shared void addFailure( Throwable reason, String title = "" ) { - TestState state = if ( is AssertionError reason ) then TestState.failure else TestState.error; - addOutput( TestOutput( state, reason, (system.nanoseconds - startTime)/1000000, title ) ); + TestState state; + if ( is AssertionError reason ) { + state = TestState.failure; + } + else if ( is TestSkippedException reason ) { + state = TestState.skipped; + } + else if ( is TestAbortedException reason ) { + state = TestState.aborted; + } + else { + state = TestState.error; + } + addOutput( TestOutput( state, reason, ( system.nanoseconds - startTime ) / 1000000, title ) ); } "Adds test success report to the test results." shared void addSuccess( String title ) { - addOutput( TestOutput( TestState.success, null, (system.nanoseconds - startTime)/1000000, title ) ); + addOutput( TestOutput( TestState.success, null, ( system.nanoseconds - startTime ) / 1000000, title ) ); } "Returns built test variant results." shared TestVariantResult variantResult { TestState state = if ( outs.empty ) then TestState.success else overallState; - return TestVariantResult( outs.sequence(), (system.nanoseconds - startTime)/1000000, state ); + return TestVariantResult( outs.sequence(), ( system.nanoseconds - startTime ) / 1000000, state ); } } From 4042e9941bcaf443853182dc8e6134a466b3e4a8 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 20:47:45 +0300 Subject: [PATCH 07/72] verifies if thread still is alive before call next prepost --- source/herd/asynctest/internal/CurrentThread.java | 10 ++++++++++ .../herd/asynctest/rule/ChainedPrePostContext.ceylon | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 source/herd/asynctest/internal/CurrentThread.java diff --git a/source/herd/asynctest/internal/CurrentThread.java b/source/herd/asynctest/internal/CurrentThread.java new file mode 100644 index 0000000..23a9551 --- /dev/null +++ b/source/herd/asynctest/internal/CurrentThread.java @@ -0,0 +1,10 @@ +package herd.asynctest.internal; + +import java.lang.Thread; + +public class CurrentThread { + public static boolean isWorks() { + Thread thr = Thread.currentThread(); + return !thr.isInterrupted() && thr.isAlive(); + } +} diff --git a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon index 256e0b3..119e250 100644 --- a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon +++ b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon @@ -13,6 +13,9 @@ import java.util.concurrent.locks { Condition, ReentrantLock } +import herd.asynctest.internal { + CurrentThread +} "Provides in chain calls of the given functions." @@ -54,7 +57,7 @@ class ChainedPrePostContext( AsyncPrePostContext context, Iterator Date: Mon, 28 Nov 2016 20:47:55 +0300 Subject: [PATCH 08/72] docs --- source/herd/asynctest/rule/package.ceylon | 19 ++----------------- source/herd/asynctest/runner/package.ceylon | 2 ++ 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/source/herd/asynctest/rule/package.ceylon b/source/herd/asynctest/rule/package.ceylon index c882bbc..465e02d 100644 --- a/source/herd/asynctest/rule/package.ceylon +++ b/source/herd/asynctest/rule/package.ceylon @@ -79,23 +79,8 @@ ### Build-in test rules: - * [[AtomicValueRule]] provides atomic operations with some value. - The value is re-initialized before _each_ test. - * [[ContextualRule]] stores values local to the current thread. - This means each thread get its own copy of the value. The value is re-initialized before _each_ test. - * [[CounterRule]] provides an atomic counter reseted to initial value before _each_ test. - * [[GaugeStatement]] verifies an element against given `matcher` each time when gauged. - * [[LockAccessRule]] tool for controlling access to a shared resource by multiple threads. - * [[MeterRule]] collects statistic data on an execution time and on a rate (per second) at which a set of events occur. - * [[ResourceRule]] represents a file packaged within a module and loaded before _all_ tests started. - * [[StatisticRule]] provides statistics data for some variate values. - * [[SuiteRuleChain]] and [[TestRuleChain]] are intended to execute rules in the given order. - * [[TemporaryDirectoryRule]] represents a temporary directory which is created before _each_ test and destroyed after. - * [[TemporaryFileRule]] represents a temporary file which is created before _each_ test and destroyed after. - * [[Verifier]] extracts value from source function after _each_ test and verifies it with given matcher. - * [[VerifyRule]] extends [[AtomicValueRule]] and additionally verifies the stored value against a given matcher - after _each_ test. - + see, the package members. + Why every build-in test rule implements `non-default` rule methods? Since each method calls [[herd.asynctest::AsyncPrePostContext.proceed]] or [[herd.asynctest::AsyncPrePostContext.abort]] which complete initialization / disposing. Delegation should be used instead of extending. diff --git a/source/herd/asynctest/runner/package.ceylon b/source/herd/asynctest/runner/package.ceylon index 6081e0f..be5aac5 100644 --- a/source/herd/asynctest/runner/package.ceylon +++ b/source/herd/asynctest/runner/package.ceylon @@ -29,6 +29,8 @@ > Test runner executes only test function. All `before`, `after` and `testRule` (including `TestStatement`) callbacks are executed outside the runner. If overall execution cycle has to be repeated, see [[herd.asynctest::retry]]. + > The test function submitted to the runner blocks current thread until `context.complete` called. + #### Built-in runners From a0c772e53fa365ecd848318c20f44b7611cb84f9 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 21:36:00 +0300 Subject: [PATCH 09/72] test statement chain --- .../rule/ChainedPrePostContext.ceylon | 2 +- .../asynctest/rule/ChainedTestContext.ceylon | 76 +++++++++++++++++++ source/herd/asynctest/rule/RuleChain.ceylon | 44 ++++++----- 3 files changed, 102 insertions(+), 20 deletions(-) create mode 100644 source/herd/asynctest/rule/ChainedTestContext.ceylon diff --git a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon index 119e250..54f76fd 100644 --- a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon +++ b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon @@ -18,7 +18,7 @@ import herd.asynctest.internal { } -"Provides in chain calls of the given functions." +"Provides in chain calls of the given prepost functions." since( "0.6.0" ) by( "Lis" ) class ChainedPrePostContext( AsyncPrePostContext context, Iterator functions ) { diff --git a/source/herd/asynctest/rule/ChainedTestContext.ceylon b/source/herd/asynctest/rule/ChainedTestContext.ceylon new file mode 100644 index 0000000..ae73e52 --- /dev/null +++ b/source/herd/asynctest/rule/ChainedTestContext.ceylon @@ -0,0 +1,76 @@ +import herd.asynctest { + AsyncTestContext +} +import java.util.concurrent.locks { + Condition, + ReentrantLock +} +import java.util.concurrent.atomic { + AtomicBoolean +} +import herd.asynctest.internal { + CurrentThread +} + + +"Provides in chain calls of the given test functions." +since( "0.6.1" ) by( "Lis" ) +class ChainedTestContext ( + "Context to delegate reports." AsyncTestContext context, + "The functions to be executed." Iterator functions ) { + + "Notify to run next function." + ReentrantLock locker = ReentrantLock(); + Condition completed = locker.newCondition(); + + + "Boxing the context in order to restrict functions callbacks to completed context." + class TestContextBox() satisfies AsyncTestContext { + AtomicBoolean runningAtomic = AtomicBoolean( true ); + shared Boolean running => runningAtomic.get(); + + shared actual void complete( String title ) { + if ( runningAtomic.compareAndSet( true, false ) ) { + locker.lock(); + try { completed.signal(); } + finally { locker.unlock(); } + } + } + + shared actual void fail( Throwable|Anything() exceptionSource, String title ) { + if ( running ) { context.fail( exceptionSource, title ); } + } + + shared actual void succeed( String message ) { + if ( running ) { context.succeed( message ); } + } + } + + + void processFunctions() { + while ( is Anything(AsyncTestContext) f = functions.next(), CurrentThread.works ) { + // execute next chained function + value box = TestContextBox(); + try { f( box ); } + catch ( Throwable err ) { + box.fail( err ); + box.complete(); + } + if ( box.running ) { + // await completion + if ( locker.tryLock() ) { + try { completed.await(); } + finally { locker.unlock(); } + } + } + } + } + + + "Processes the functions execution." + shared void process() { + processFunctions(); + context.complete(); + } + +} diff --git a/source/herd/asynctest/rule/RuleChain.ceylon b/source/herd/asynctest/rule/RuleChain.ceylon index 16ea5f0..f2aa215 100644 --- a/source/herd/asynctest/rule/RuleChain.ceylon +++ b/source/herd/asynctest/rule/RuleChain.ceylon @@ -1,9 +1,10 @@ import herd.asynctest { - AsyncPrePostContext + AsyncPrePostContext, + AsyncTestContext } -"Chain of suite rules is indended to apply suite rules in specific order. +"Applies suite rules in the given order. Initialization (i.e. `initialize` methods) is performed with iterating of the `rules` in direct order, while diposing (i.e. `dispose` methods) is performed in reverse order. So, the first initialized is the last disposed." @@ -16,20 +17,16 @@ shared class SuiteRuleChain ( Anything(AsyncPrePostContext)[] cleaners = [for ( r in rules ) r.dispose ].reversed; - shared actual void dispose( AsyncPrePostContext context ) { - ChainedPrePostContext c = ChainedPrePostContext( context, cleaners.iterator() ); - c.start(); - } + shared actual void dispose( AsyncPrePostContext context ) + => ChainedPrePostContext( context, cleaners.iterator() ).start(); - shared actual void initialize( AsyncPrePostContext context ) { - ChainedPrePostContext c = ChainedPrePostContext( context, initializers.iterator() ); - c.start(); - } + shared actual void initialize( AsyncPrePostContext context ) + => ChainedPrePostContext( context, initializers.iterator() ).start(); } -"Chain of test rules is indended to apply test rules in specific order. +"Applies test rules in the given order. Initialization (i.e. `before` methods) is performed with iterating of the `rules` in direct order, while diposing (i.e. `after` methods) is performed in reverse order. So, the first initialized is the last disposed." @@ -42,14 +39,23 @@ shared class TestRuleChain ( Anything(AsyncPrePostContext)[] cleaners = [for ( r in rules ) r.after ].reversed; - shared actual void after( AsyncPrePostContext context ) { - ChainedPrePostContext c = ChainedPrePostContext( context, cleaners.iterator() ); - c.start(); - } + shared actual void after( AsyncPrePostContext context ) + => ChainedPrePostContext( context, cleaners.iterator() ).start(); - shared actual void before( AsyncPrePostContext context ) { - ChainedPrePostContext c = ChainedPrePostContext( context, initializers.iterator() ); - c.start(); - } + shared actual void before( AsyncPrePostContext context ) + => ChainedPrePostContext( context, initializers.iterator() ).start(); } + + +"Applies test statements in the given order." +tagged( "TestStatement" ) since( "0.6.1" ) by( "Lis" ) +shared class TestStatementChain ( + "A list of the statements to be applied in order they are provided." TestStatement* statements +) satisfies TestStatement { + + Anything(AsyncTestContext)[] statementFunctions = [for ( s in statements ) s.apply ]; + + shared actual void apply( AsyncTestContext context ) + => ChainedTestContext( context, statementFunctions.iterator() ).process(); +} From 91ed2b63766677ce4bf024b1c0028912ce35ab6a Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 23:08:19 +0300 Subject: [PATCH 10/72] doc --- source/herd/asynctest/AsyncMessageContext.ceylon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/herd/asynctest/AsyncMessageContext.ceylon b/source/herd/asynctest/AsyncMessageContext.ceylon index 7ddf5bc..89c1a69 100644 --- a/source/herd/asynctest/AsyncMessageContext.ceylon +++ b/source/herd/asynctest/AsyncMessageContext.ceylon @@ -1,7 +1,7 @@ "Base interface to push test messages to. - The interface is mainly used by test runners [[herd.asynctest.runner::AsyncTestRunner]]. + The interface is mainly used by test runners, see package [[package herd.asynctest.runner]] for details. Test function receives derived interface [[AsyncTestContext]]." since( "0.6.0" ) by( "Lis" ) shared interface AsyncMessageContext { From 58991747449bd285d914d66039218a5ba8b6d484 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 23:20:14 +0300 Subject: [PATCH 11/72] rearrange annotation docs --- source/herd/asynctest/annotations.ceylon | 159 +++++++++--------- .../herd/asynctest/runner/annotations.ceylon | 16 +- 2 files changed, 88 insertions(+), 87 deletions(-) diff --git a/source/herd/asynctest/annotations.ceylon b/source/herd/asynctest/annotations.ceylon index 364403b..26c0832 100644 --- a/source/herd/asynctest/annotations.ceylon +++ b/source/herd/asynctest/annotations.ceylon @@ -26,9 +26,10 @@ import herd.asynctest.runner { } -"The same as `testExecutor(`\`class AsyncTestExecutor\``)`" +"The same as `testExecutor(`\`class AsyncTestExecutor\``)`." +see( `class AsyncTestExecutor` ) since( "0.6.0" ) by( "Lis" ) -shared annotation TestExecutorAnnotation async() => testExecutor(`class AsyncTestExecutor`); +shared annotation TestExecutorAnnotation async() => testExecutor( `class AsyncTestExecutor` ); "Calls [[source]] to get source value." @@ -48,25 +49,6 @@ Result extractSourceValue( FunctionOrValueDeclaration source, Object? in } -"Annotation class for [[arguments]]." -see( `function arguments` ) -since( "0.5.0" ) by( "Lis" ) -shared final annotation class ArgumentsAnnotation ( - "The source function or value declaration which has to take no arguments and has to return a stream of values. - The source may be either top-level or tested class shared member." - shared FunctionOrValueDeclaration source -) - satisfies OptionalAnnotation -{ - - "Calls [[source]] to get argument stream." - shared Anything[] argumentList ( - "Instance of the test class or `null` if test is performed using top-level function." Object? instance - ) => extractSourceValue( source, instance ); - -} - - "Indicates that test container class or test prepost function have to be instantiated or called using arguments provided by this annotation, see [[ArgumentsAnnotation.argumentList]]. @@ -85,40 +67,35 @@ shared final annotation class ArgumentsAnnotation ( } " +see( `function arguments` ) since( "0.5.0" ) by( "Lis" ) -shared annotation ArgumentsAnnotation arguments ( +shared final annotation class ArgumentsAnnotation ( "The source function or value declaration which has to take no arguments and has to return a stream of values. The source may be either top-level or tested class shared member." - FunctionOrValueDeclaration source -) - => ArgumentsAnnotation( source ); - - - -"Annotation class for [[parameterized]]." -since( "0.6.0" ) by( "Lis" ) -see( `function parameterized`, `class TestVariant` ) -shared final annotation class ParameterizedAnnotation ( - "The source function or value declaration which has to take no arguments and has to return a stream - of test variants: `{TestVariant*}`. - The source may be either top-level or tested class shared member." - shared FunctionOrValueDeclaration source, - "Maximum number of failed variants before stop. Unlimited if <= 0." - shared Integer maxFailedVariants + shared FunctionOrValueDeclaration source ) - satisfies SequencedAnnotation & TestVariantProvider + satisfies OptionalAnnotation { - "Returns test variant enumerator based on test variants extracted from `source`." - shared actual TestVariantEnumerator variants( FunctionDeclaration testFunction, Object? instance) - => TestVariantIterator ( - extractSourceValue<{TestVariant*}>( source, instance ).iterator().next, - maxFailedVariants - ); + "Calls [[source]] to get argument stream." + shared Anything[] argumentList ( + "Instance of the test class or `null` if test is performed using top-level function." Object? instance + ) => extractSourceValue( source, instance ); } +"Provides arguments for a one-shot functions. See [[ArgumentsAnnotation]] for details." +since( "0.5.0" ) by( "Lis" ) +shared annotation ArgumentsAnnotation arguments ( + "The source function or value declaration which has to take no arguments and has to return a stream of values. + The source may be either top-level or tested class shared member." + FunctionOrValueDeclaration source +) + => ArgumentsAnnotation( source ); + + + "Indicates that generic (with possibly empty generic parameter list) test function has to be executed with given test variants. @@ -183,6 +160,30 @@ shared final annotation class ParameterizedAnnotation ( According to second argument of `parameterized` annotation the test will be stopped if two different invoking of `thereAndBackAgain` with two different arguments report failure. " +since( "0.6.0" ) by( "Lis" ) +see( `function parameterized`, `class TestVariant` ) +shared final annotation class ParameterizedAnnotation ( + "The source function or value declaration which has to take no arguments and has to return a stream + of test variants: `{TestVariant*}`. + The source may be either top-level or tested class shared member." + shared FunctionOrValueDeclaration source, + "Maximum number of failed variants before stop. Unlimited if <= 0." + shared Integer maxFailedVariants +) + satisfies SequencedAnnotation & TestVariantProvider +{ + + "Returns test variant enumerator based on test variants extracted from `source`." + shared actual TestVariantEnumerator variants( FunctionDeclaration testFunction, Object? instance) + => TestVariantIterator ( + extractSourceValue<{TestVariant*}>( source, instance ).iterator().next, + maxFailedVariants + ); + +} + + +"Provides parameters for the parameterized testing. See [[ParameterizedAnnotation]] for the details." see( `class TestVariant` ) since( "0.6.0" ) by( "Lis" ) shared annotation ParameterizedAnnotation parameterized ( @@ -196,19 +197,6 @@ shared annotation ParameterizedAnnotation parameterized ( => ParameterizedAnnotation( source, maxFailedVariants ); -"Annotation class for [[factory]]." -see( `function factory`, `interface AsyncFactoryContext` ) -since( "0.6.0" ) by( "Lis" ) -shared final annotation class FactoryAnnotation ( - "Function used to instantiate anotated class. - Has to take no arguments or just a one argument of [[AsyncFactoryContext]] type. - Has to return an instance of the looking class." - shared FunctionDeclaration factoryFunction -) - satisfies OptionalAnnotation -{} - - "Indicates that class has to be instantiated using a given factory function. [[factory]] annotation takes declaration of top-level factory function. @@ -246,6 +234,19 @@ shared final annotation class FactoryAnnotation ( > Asynchronous version has to call [[AsyncFactoryContext.fill]] or [[AsyncFactoryContext.abort]]. > Synchronous version has to return non-optional object or throw. " +see( `function factory`, `interface AsyncFactoryContext` ) +since( "0.6.0" ) by( "Lis" ) +shared final annotation class FactoryAnnotation ( + "Function used to instantiate anotated class. + Has to take no arguments or just a one argument of [[AsyncFactoryContext]] type. + Has to return an instance of the looking class." + shared FunctionDeclaration factoryFunction +) + satisfies OptionalAnnotation +{} + + +"Provides factory function for an object instantiation. See [[FactoryAnnotation]] for the details." see( `interface AsyncFactoryContext` ) since( "0.6.0" ) by( "Lis" ) shared annotation FactoryAnnotation factory ( @@ -257,14 +258,6 @@ shared annotation FactoryAnnotation factory ( ) => FactoryAnnotation( factoryFunction ); -"Annotation class for [[concurrent]]." -see( `function concurrent` ) -since( "0.6.0" ) by( "Lis" ) -shared final annotation class ConcurrentAnnotation() - satisfies OptionalAnnotation -{} - - "Indicates that all test suites (each suite contains all top-level test functions in the given package or all test methods of the given class) contained in the marked container have to be executed in concurrent mode. The functions within each suite are executed sequentially in a one thread while the suites are executed concurrently @@ -272,11 +265,21 @@ shared final annotation class ConcurrentAnnotation() > Thread pool with fixed number of threads equals to number of available processors (cores) is used to execute tests in concurrent mode." +see( `function concurrent` ) +since( "0.6.0" ) by( "Lis" ) +shared final annotation class ConcurrentAnnotation() + satisfies OptionalAnnotation +{} + + +"Indicates that all test suites contained in the marked container have to be executed in concurrent mode." since( "0.6.0" ) by( "Lis" ) shared annotation ConcurrentAnnotation concurrent() => ConcurrentAnnotation(); -"Annotation class for [[timeout]]." +"Indicates that if test function execution takes more than `timeoutMilliseconds` the test has to be interrupted. + The annotation is applied to any function called using [[AsyncTestExecutor]]: prepost functions, test rules, + factory and test functions." see( `function timeout` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class TimeoutAnnotation( "Timeout in milliseconds." shared Integer timeoutMilliseconds ) @@ -285,16 +288,20 @@ shared final annotation class TimeoutAnnotation( "Timeout in milliseconds." shar {} -"Indicates that if test function execution takes more than `timeoutMilliseconds` the test has to be interrupted. - The annotation is applied to any function called using [[AsyncTestExecutor]]: prepost functions, test rules, - factory and test functions." +"Indicates that if test function execution takes more than `timeoutMilliseconds` the test has to be interrupted." since( "0.6.0" ) by( "Lis" ) shared annotation TimeoutAnnotation timeout( "Timeout in milliseconds." Integer timeoutMilliseconds ) => TimeoutAnnotation( timeoutMilliseconds ); -"Annotation class for [[retry]]." -see( `function retry` ) +"Indicates that execution of a test function or all test functions within annotated test container + have to be retryed using the given `RepeatStrategy` (extracted from `source`). + + > Overall execution cycle including `before`, `after` and `testRule` callbacks are repeated! + + If you need to repeat just test function execution, look on [[RepeatRunner]]. + " +see( `function retry`, `class RepeatRunner`, `interface RepeatStrategy` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class RetryAnnotation ( "Repeat strategy source which has to take no arguments and has to return instance of [[RepeatStrategy]] type. @@ -305,13 +312,7 @@ shared final annotation class RetryAnnotation ( {} -"Indicates that test function or all functions within test container have to be tested using the given - `RepeatStrategy` (extracted from `source`). - - > Overall execution cycle including `before`, `after` and `testRule` callbacks are repeated! - - If you need to repeat just test function execution, look on [[RepeatRunner]]. - " +"Provides retry trategy for the tet function execution. See details in [[RetryAnnotation]]." see( `class RepeatRunner`, `interface RepeatStrategy` ) since( "0.6.0" ) by( "Lis" ) shared annotation RetryAnnotation retry ( diff --git a/source/herd/asynctest/runner/annotations.ceylon b/source/herd/asynctest/runner/annotations.ceylon index c0cdea1..ec9ed1c 100644 --- a/source/herd/asynctest/runner/annotations.ceylon +++ b/source/herd/asynctest/runner/annotations.ceylon @@ -7,8 +7,13 @@ import ceylon.language.meta.declaration { } -"Annotation class for [[runWith]]." -see( `function runWith` ) +"Indicates that the test function has to be run with the given runner. + The runner has to satisfy [[AsyncTestRunner]] interface. + + If class or top-level container is marked with the annotation each test function of the container + is executed with the given runner. + " +see( `function runWith`, `package herd.asynctest.runner`, `interface AsyncTestRunner` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class RunWithAnnotation ( "Runner source. Top-level function or value or method or attribute of a test function container. @@ -19,12 +24,7 @@ shared final annotation class RunWithAnnotation ( {} -"Indicates that the test function has to be run with the given runner. - The runner has to satisfy [[AsyncTestRunner]] interface. - - If class or top-level container is marked with the annotation each test function of the container - is executed with the given runner. - " +"Provides test runner for the annotated test function. See details in [[RunWithAnnotation]]." since( "0.6.0" ) by( "Lis" ) see( `package herd.asynctest.runner`, `interface AsyncTestRunner` ) shared annotation RunWithAnnotation runWith ( From d8edfa1aaefded81ea9e4d8814cbe5c33c5f9275 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 23:23:58 +0300 Subject: [PATCH 12/72] doc --- source/herd/asynctest/match/checkMatchers.ceylon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/herd/asynctest/match/checkMatchers.ceylon b/source/herd/asynctest/match/checkMatchers.ceylon index 53a093e..f2022e3 100644 --- a/source/herd/asynctest/match/checkMatchers.ceylon +++ b/source/herd/asynctest/match/checkMatchers.ceylon @@ -46,8 +46,8 @@ shared class Identical ( * trasitivity, if x==y and y==z then x==z * \'hashivity\', if x==y then x.hash==y.hash - In order to have value to compare to `clone` function is used which has to return new object - which has to be equal to the given one. + In order to have value to compare to `clone` function is used. The function has to return new object + which has to be equal to the passed to. " tagged( "Checkers" ) since( "0.6.0" ) by( "Lis" ) shared class ValueEquality ( From 8af7368aeeded4759f861b93aecdafbc319de653 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 28 Nov 2016 23:26:48 +0300 Subject: [PATCH 13/72] docs --- source/herd/asynctest/runner/RepeatStrategy.ceylon | 1 + 1 file changed, 1 insertion(+) diff --git a/source/herd/asynctest/runner/RepeatStrategy.ceylon b/source/herd/asynctest/runner/RepeatStrategy.ceylon index b32eeb2..d258d39 100644 --- a/source/herd/asynctest/runner/RepeatStrategy.ceylon +++ b/source/herd/asynctest/runner/RepeatStrategy.ceylon @@ -30,6 +30,7 @@ shared interface RepeatStrategy { tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) shared object repeatOnce satisfies RepeatStrategy { + "Always returns passed variant." shared actual TestVariantResult? completeOrRepeat( TestVariantResult variant ) => variant; } From b5f1f26cac4291797060ee29104446c00a96fc6c Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 29 Nov 2016 19:08:19 +0300 Subject: [PATCH 14/72] refactor rules --- .../asynctest/rule/AtomicValueRule.ceylon | 6 ++---- source/herd/asynctest/rule/CounterRule.ceylon | 4 ++-- .../herd/asynctest/rule/LockAccessRule.ceylon | 21 +++++++++++++++++-- source/herd/asynctest/rule/SignalRule.ceylon | 12 +++++------ .../herd/asynctest/rule/ruleAnnotation.ceylon | 4 +++- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/source/herd/asynctest/rule/AtomicValueRule.ceylon b/source/herd/asynctest/rule/AtomicValueRule.ceylon index 6f8726c..6cf3166 100644 --- a/source/herd/asynctest/rule/AtomicValueRule.ceylon +++ b/source/herd/asynctest/rule/AtomicValueRule.ceylon @@ -16,7 +16,7 @@ shared class AtomicValueRule( "Initial value source." Element|Element() satisfies TestRule { - variable Atomic storage = if ( is Element() initial ) + Atomic storage = if ( is Element() initial ) then instantiateAtomic( initial() ) else instantiateAtomic( initial ); @@ -38,9 +38,7 @@ shared class AtomicValueRule( "Initial value source." Element|Element() shared actual void after( AsyncPrePostContext context ) => context.proceed(); shared actual void before( AsyncPrePostContext context ) { - storage = if ( is Element() initial ) - then instantiateAtomic( initial() ) - else instantiateAtomic( initial ); + sense = if ( is Element() initial ) then initial() else initial; context.proceed(); } diff --git a/source/herd/asynctest/rule/CounterRule.ceylon b/source/herd/asynctest/rule/CounterRule.ceylon index ca7be0c..394803c 100644 --- a/source/herd/asynctest/rule/CounterRule.ceylon +++ b/source/herd/asynctest/rule/CounterRule.ceylon @@ -10,7 +10,7 @@ import java.util.concurrent.atomic { tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class CounterRule( "Initial value of the counter." Integer initial = 0 ) satisfies TestRule { - variable AtomicLong atomicCounter = AtomicLong( initial ); + AtomicLong atomicCounter = AtomicLong( initial ); "The current value of the counter." @@ -33,7 +33,7 @@ shared class CounterRule( "Initial value of the counter." Integer initial = 0 ) shared actual void after( AsyncPrePostContext context ) => context.proceed(); shared actual void before( AsyncPrePostContext context ) { - atomicCounter = AtomicLong( initial ); + atomicCounter.set( initial ); context.proceed(); } diff --git a/source/herd/asynctest/rule/LockAccessRule.ceylon b/source/herd/asynctest/rule/LockAccessRule.ceylon index ac7e280..d4c1bfe 100644 --- a/source/herd/asynctest/rule/LockAccessRule.ceylon +++ b/source/herd/asynctest/rule/LockAccessRule.ceylon @@ -4,6 +4,9 @@ import herd.asynctest { import java.util.concurrent.locks { ReentrantLock } +import java.util.concurrent.atomic { + AtomicInteger +} "Tool for controlling access to a shared resource of `Element` type by multiple threads. @@ -38,13 +41,27 @@ shared class LockAccessRule ( { class Box( shared variable Element elem ) { + shared AtomicInteger lockCountAtomic = AtomicInteger( 0 ); + shared AtomicInteger unlockCountAtomic = AtomicInteger( 0 ); ReentrantLock locker = ReentrantLock(); - shared void lock() => locker.lock(); - shared void unlock() => locker.unlock(); + shared void lock() { + locker.lock(); + lockCountAtomic.incrementAndGet(); + } + shared void unlock() { + unlockCountAtomic.incrementAndGet(); + locker.unlock(); + } } variable Box stored = Box( if ( is Element() initial ) then initial() else initial ); + "Total number of times the lock has been obtained." + shared Integer lockCount => stored.lockCountAtomic.get(); + + "Total number of times the lock has been released." + shared Integer unlockCount => stored.unlockCountAtomic.get(); + "Locks the resource and provides access to. Actual resource is stored when `release` is called, but not when `element` is modified." diff --git a/source/herd/asynctest/rule/SignalRule.ceylon b/source/herd/asynctest/rule/SignalRule.ceylon index cf516ee..59a3edc 100644 --- a/source/herd/asynctest/rule/SignalRule.ceylon +++ b/source/herd/asynctest/rule/SignalRule.ceylon @@ -49,9 +49,9 @@ shared class SignalRule() satisfies TestRule { variable Condition condition = lock.newCondition(); // counting of await, signal and signal all operations - variable AtomicInteger awaitCounts = AtomicInteger( 0 ); - variable AtomicInteger signalCounts = AtomicInteger( 0 ); - variable AtomicInteger signalAllCounts = AtomicInteger( 0 ); + AtomicInteger awaitCounts = AtomicInteger( 0 ); + AtomicInteger signalCounts = AtomicInteger( 0 ); + AtomicInteger signalAllCounts = AtomicInteger( 0 ); "Number of times the `await` has been called. @@ -70,9 +70,9 @@ shared class SignalRule() satisfies TestRule { } shared actual void before( AsyncPrePostContext context ) { - awaitCounts = AtomicInteger( 0 ); - signalCounts = AtomicInteger( 0 ); - signalAllCounts = AtomicInteger( 0 ); + awaitCounts.set( 0 ); + signalCounts.set( 0 ); + signalAllCounts.set( 0 ); lock = ReentrantLock(); condition = lock.newCondition(); context.proceed(); diff --git a/source/herd/asynctest/rule/ruleAnnotation.ceylon b/source/herd/asynctest/rule/ruleAnnotation.ceylon index 10a0a09..f7d23ff 100644 --- a/source/herd/asynctest/rule/ruleAnnotation.ceylon +++ b/source/herd/asynctest/rule/ruleAnnotation.ceylon @@ -3,7 +3,9 @@ import ceylon.language.meta.declaration { } -"Annotation class for [[testRule]]." +"Indicates that the annotated value or attribute identifies a test rule. + The value declaration has to satisfy [[SuiteRule]], [[TestRule]] or [[TestStatement]] interfaces." +see( `interface TestRule`, `interface SuiteRule`, `interface TestStatement` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class TestRuleAnnotation() satisfies OptionalAnnotation From b19368bdf45ba67d6cd20c52b1189fa22e2e81fe Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 29 Nov 2016 22:26:40 +0300 Subject: [PATCH 15/72] AtomicLong instead of AtomicInteger --- source/herd/asynctest/rule/LockAccessRule.ceylon | 6 +++--- source/herd/asynctest/rule/SignalRule.ceylon | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/herd/asynctest/rule/LockAccessRule.ceylon b/source/herd/asynctest/rule/LockAccessRule.ceylon index d4c1bfe..8c79c02 100644 --- a/source/herd/asynctest/rule/LockAccessRule.ceylon +++ b/source/herd/asynctest/rule/LockAccessRule.ceylon @@ -5,7 +5,7 @@ import java.util.concurrent.locks { ReentrantLock } import java.util.concurrent.atomic { - AtomicInteger + AtomicLong } @@ -41,8 +41,8 @@ shared class LockAccessRule ( { class Box( shared variable Element elem ) { - shared AtomicInteger lockCountAtomic = AtomicInteger( 0 ); - shared AtomicInteger unlockCountAtomic = AtomicInteger( 0 ); + shared AtomicLong lockCountAtomic = AtomicLong( 0 ); + shared AtomicLong unlockCountAtomic = AtomicLong( 0 ); ReentrantLock locker = ReentrantLock(); shared void lock() { locker.lock(); diff --git a/source/herd/asynctest/rule/SignalRule.ceylon b/source/herd/asynctest/rule/SignalRule.ceylon index 59a3edc..026c543 100644 --- a/source/herd/asynctest/rule/SignalRule.ceylon +++ b/source/herd/asynctest/rule/SignalRule.ceylon @@ -9,7 +9,7 @@ import java.util.concurrent { TimeUnit } import java.util.concurrent.atomic { - AtomicInteger + AtomicLong } @@ -49,9 +49,9 @@ shared class SignalRule() satisfies TestRule { variable Condition condition = lock.newCondition(); // counting of await, signal and signal all operations - AtomicInteger awaitCounts = AtomicInteger( 0 ); - AtomicInteger signalCounts = AtomicInteger( 0 ); - AtomicInteger signalAllCounts = AtomicInteger( 0 ); + AtomicLong awaitCounts = AtomicLong( 0 ); + AtomicLong signalCounts = AtomicLong( 0 ); + AtomicLong signalAllCounts = AtomicLong( 0 ); "Number of times the `await` has been called. From dc215237f4a6d6bd944d484c957be71798c81534 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 30 Nov 2016 00:30:13 +0300 Subject: [PATCH 16/72] stringified --- source/herd/asynctest/internal/Atomic.ceylon | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/herd/asynctest/internal/Atomic.ceylon b/source/herd/asynctest/internal/Atomic.ceylon index 6fd884a..c85cc6b 100644 --- a/source/herd/asynctest/internal/Atomic.ceylon +++ b/source/herd/asynctest/internal/Atomic.ceylon @@ -71,6 +71,8 @@ class AtomicBoolean( Boolean initial ) satisfies Atomic shared actual Boolean getAndSet( Boolean newValue ) => storage.getAndSet( newValue ); shared actual void set( Boolean v ) => storage.set( v ); + + string => "atomic boolean of '``get()``'"; } @@ -87,6 +89,8 @@ class AtomicInteger( Integer initial ) satisfies Atomic shared actual Integer getAndSet( Integer newValue ) => storage.getAndSet( newValue ); shared actual void set( Integer v ) => storage.set( v ); + + string => "atomic integer of '``get()``'"; } @@ -105,6 +109,8 @@ class AtomicFloat( Float initial ) satisfies Atomic => Double.longBitsToDouble( storage.getAndSet( Double.doubleToRawLongBits( newValue ) ) ); shared actual void set( Float v ) => storage.set( Double.doubleToRawLongBits( v ) ); + + string => "atomic float of '``get()``'"; } @@ -121,5 +127,7 @@ class AtomicReference( Element initial ) satisfies Atomic shared actual Element getAndSet( Element newValue ) => storage.getAndSet( newValue ); shared actual void set( Element v ) => storage.set( v ); + + string => "atomic reference of '``stringify( get() )``'"; } From 3305fedf9241f774b4c5f23af03b1b2201119c45 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 30 Nov 2016 00:30:54 +0300 Subject: [PATCH 17/72] doc --- .../asynctest/fibonacci/fibonacciTest.ceylon | 6 +-- .../CeylonJavaMapMicrobenchmark.ceylon | 39 ++----------------- 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon b/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon index d8f84b2..941e616 100644 --- a/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon +++ b/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon @@ -38,8 +38,8 @@ see( `function runFibonacciTest` ) The function is marked with `testExecutor` annotation in order to perform asynchronous test. Alternatively `testExecutor` annotation can be used at module level." -test async parameterized( `value fibonacciNumbers` ) -timeout( 5000 ) +test async parameterized(`value fibonacciNumbers`) +timeout(5000) shared void runFibonacciTest ( "Context to send test results." AsyncTestContext context, "Index of Fibonacci number to be calculated." Integer indexOfFibonacciNumber, @@ -50,7 +50,7 @@ shared void runFibonacciTest ( (Integer val) { context.assertThat ( val, - EqualTo( expectedFibonacciNumber ).and( Mapping( fibonacciNumberIndex, EqualTo( indexOfFibonacciNumber ) ) ), + EqualTo(expectedFibonacciNumber).and(Mapping(fibonacciNumberIndex, EqualTo(indexOfFibonacciNumber))), "", true ); context.complete(); diff --git a/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon b/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon index 1ce824a..c3a72a9 100644 --- a/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon +++ b/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon @@ -8,8 +8,7 @@ import herd.asynctest { import ceylon.test { test, - afterTestRun, - beforeTestRun + afterTestRun } import ceylon.collection { @@ -122,7 +121,7 @@ class CeylonJavaMapMicrobenchmark( Float tolerance ) shared testRule MeterRule removeCeylon = MeterRule(); shared testRule MeterRule removeJava = MeterRule(); - //black hole inorder to eliminate dead-code of `get` method return + //black hole in order to eliminate dead-code of `get` method return shared testRule AtomicValueRule blackHole = AtomicValueRule( 0 ); @@ -139,7 +138,7 @@ class CeylonJavaMapMicrobenchmark( Float tolerance ) () => removeCeylon.timeStatistic.mean / removeJava.timeStatistic.mean, LessOrEqual( 1.0 + tolerance ), "'remove' Ceylon / Java ratio", true ); - + afterTestRun shared void dispose() { plotReporter.report ( @@ -147,37 +146,7 @@ class CeylonJavaMapMicrobenchmark( Float tolerance ) hashMapRatioChart.build(), treeMapRatioChart.build() ); } - - beforeTestRun shared void warmUp() { - // Ceylon HashMap - chartMapTest ( - 1000, 10, 0.2, - CeylonMapWrapper( HashMap() ), - ceylonHashPutPlotter, ceylonHashGetPlotter, ceylonHashRemovePlotter, - putCeylon, getCeylon, removeCeylon - ); - // Java HashMap - chartMapTest ( - 1000, 10, 0.2, - JavaMapWrapper( JHashMap() ), - javaHashPutPlotter, javaHashGetPlotter, javaHashRemovePlotter, - putJava, getJava, removeJava - ); - // Ceylon TreeMap - chartMapTest ( - 1000, 10, 0.2, - CeylonMapWrapper( TreeMap( increasing ) ), - ceylonTreePutPlotter, ceylonTreeGetPlotter, ceylonTreeRemovePlotter, - putCeylon, getCeylon, removeCeylon - ); - // Java TreeMap - chartMapTest ( - 1000, 10, 0.2, - JavaMapWrapper( JTreeMap( stringComparator ) ), - javaTreePutPlotter, javaTreeGetPlotter, javaTreeRemovePlotter, - putJava, getJava, removeJava - ); - } + "Runs `HashMap` test. Compares performance Ceylon `HashMap` to Java one. Test is performed using `chartMapTest`. From 32f9829aa72008ed56524ad6ed7afac814590391 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 30 Nov 2016 00:34:03 +0300 Subject: [PATCH 18/72] rule which prevents access to value from inproperly completed tests --- .../herd/asynctest/AsyncTestProcessor.ceylon | 2 + .../internal/ContextThreadGroup.ceylon | 21 ++++-- .../internal/MarkerThreadGroup.ceylon | 10 +++ .../asynctest/rule/CurrentTestStore.ceylon | 73 +++++++++++++++++++ source/herd/asynctest/rule/exceptions.ceylon | 7 ++ 5 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 source/herd/asynctest/internal/MarkerThreadGroup.ceylon create mode 100644 source/herd/asynctest/rule/CurrentTestStore.ceylon create mode 100644 source/herd/asynctest/rule/exceptions.ceylon diff --git a/source/herd/asynctest/AsyncTestProcessor.ceylon b/source/herd/asynctest/AsyncTestProcessor.ceylon index a3433ff..f88ee67 100644 --- a/source/herd/asynctest/AsyncTestProcessor.ceylon +++ b/source/herd/asynctest/AsyncTestProcessor.ceylon @@ -119,6 +119,8 @@ class AsyncTestProcessor( TestInfo testInfo = TestInfo ( testFunctionDeclaration, variant.parameters, variant.arguments, variant.variantName, timeOutMilliseconds ); + // test with next ID is started + group.incrementTestID(); // run initializers firstly if ( nonempty initErrs = prePostContext.run( intializers, testInfo ) ) { // initialization has been failed diff --git a/source/herd/asynctest/internal/ContextThreadGroup.ceylon b/source/herd/asynctest/internal/ContextThreadGroup.ceylon index a89e295..9464579 100644 --- a/source/herd/asynctest/internal/ContextThreadGroup.ceylon +++ b/source/herd/asynctest/internal/ContextThreadGroup.ceylon @@ -1,9 +1,11 @@ import java.lang { Thread, ThreadDeath, - ThreadGroup, SecurityException } +import java.util.concurrent.atomic { + JAtomicLong = AtomicLong +} "Represents group of the context - run main function, listens uncaught exceptions and interrupt all threads." @@ -12,7 +14,9 @@ shared class ContextThreadGroup( String title ) { "Thread group." - class InternalThreadGroup() extends ThreadGroup( title ) { + class InternalThreadGroup( Integer initialTestID ) extends MarkerThreadGroup( title ) { + + shared JAtomicLong atomicTestID = JAtomicLong( initialTestID ); variable Anything( Thread, Throwable )? uncaughtExceptionListener = null; @@ -23,7 +27,9 @@ shared class ContextThreadGroup( String title ) "The group has been interrupted." shared Boolean groupInterrupted => hasBeenInterrupted; - + shared actual Integer testID => atomicTestID.get(); + + "Creates new thread or returns reused one." ReusableThread getThread() { if ( exists ret = reusedThread ) { @@ -99,12 +105,17 @@ shared class ContextThreadGroup( String title ) "Thread group." - variable InternalThreadGroup currentGroup = InternalThreadGroup(); + variable InternalThreadGroup currentGroup = InternalThreadGroup( 1 ); "Stops the current thread." shared void completeCurrent() => currentGroup.completeCurrent(); + "Increments currently run test ID. + The ID is to be a marker of the test run." + shared void incrementTestID() => currentGroup.atomicTestID.incrementAndGet(); + + "Executes `run` on separated thread belongs to this group. Awaits completion no more then `timeoutMilliseconds` or unlimited if it is <= 0." shared Boolean execute ( @@ -113,7 +124,7 @@ shared class ContextThreadGroup( String title ) "Function to be executed on separated thread." Anything() run ) { if ( currentGroup.groupInterrupted ) { - currentGroup = InternalThreadGroup(); + currentGroup = InternalThreadGroup( currentGroup.testID ); } return currentGroup.execute( uncaughtExceptionListener, timeoutMilliseconds, run ); } diff --git a/source/herd/asynctest/internal/MarkerThreadGroup.ceylon b/source/herd/asynctest/internal/MarkerThreadGroup.ceylon new file mode 100644 index 0000000..071cea2 --- /dev/null +++ b/source/herd/asynctest/internal/MarkerThreadGroup.ceylon @@ -0,0 +1,10 @@ +import java.lang { + ThreadGroup +} + +"Just a marker for the thread group - means the group is test executor group." +since( "0.6.1" ) by( "Lis" ) +shared abstract class MarkerThreadGroup( String title ) extends ThreadGroup( title ) { + "ID of the currently run test." + shared formal Integer testID; +} diff --git a/source/herd/asynctest/rule/CurrentTestStore.ceylon b/source/herd/asynctest/rule/CurrentTestStore.ceylon new file mode 100644 index 0000000..f2aca05 --- /dev/null +++ b/source/herd/asynctest/rule/CurrentTestStore.ceylon @@ -0,0 +1,73 @@ +import herd.asynctest { + AsyncPrePostContext +} +import herd.asynctest.internal { + MarkerThreadGroup, + stringify +} +import java.lang { + Thread, + ThreadGroup +} +import java.util.concurrent.atomic { + AtomicLong +} + + +"The rule which returns an element only if it is requested from the same test which initializes the rule. + Otherwise [[TestAccessDenied]] is thrown. + Actually, a thread interrupted by timeout or by something else may still be alive and may modify a rule. + This way improperly completed test may modify the currently executed test data. + `CurrentTestStore` rule is indended to prevent such behaviour throwing [[TestAccessDenied]] exception + when not owner tries to get stored element. + + > Owner here is the test for which initialization has been performed. + > The stored value is reseted by extracted from source value each time the new test is started. + " +tagged( "TestRule" ) since( "0.6.1" ) by( "Lis" ) +shared class CurrentTestStore( "Source of the stored value." Element | Element() source ) satisfies TestRule +{ + + "ID of the currently run test." + AtomicLong currentTestID = AtomicLong( -1 ); + + // TODO: has to be volatile! + "Element stored here." + variable Element storage = if ( is Element() source ) then source() else source; + + "Returns marker group the current thread belongs to." + MarkerThreadGroup? getMarkerGroup() { + variable ThreadGroup? gr = Thread.currentThread().threadGroup; + while ( exists cur = gr ) { + if ( is MarkerThreadGroup cur ) { + return cur; + } + gr = cur.parent; + } + return null; + } + + + "Returns a value if current test ownes the value or throws [[TestAccessDenied]] exception if not." + throws( `class TestAccessDenied`, "The `element` is requested from a test which doesn't own this." ) + shared Element element { + if ( exists marker = getMarkerGroup(), marker.testID == currentTestID.get() ) { + return storage; + } + else { + throw TestAccessDenied( "``stringify(storage)`` is requested from a test which doesn't own this." ); + } + } + + shared actual void after( AsyncPrePostContext context ) { + currentTestID.set( -1 ); + context.proceed(); + } + + shared actual void before( AsyncPrePostContext context ) { + storage = if ( is Element() source ) then source() else source; + currentTestID.set( getMarkerGroup()?.testID else -1 ); + context.proceed(); + } + +} diff --git a/source/herd/asynctest/rule/exceptions.ceylon b/source/herd/asynctest/rule/exceptions.ceylon new file mode 100644 index 0000000..26605b4 --- /dev/null +++ b/source/herd/asynctest/rule/exceptions.ceylon @@ -0,0 +1,7 @@ + + +"Exception thrown when access to a value is requested from a test which doesn't own the value." +see( `class CurrentTestStore` ) +since( "0.6.1" ) by( "Lis" ) +shared class TestAccessDenied( String message ) extends Exception( message ) +{} From df9433385b850f2ded45088d9171296521fa05fd Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 30 Nov 2016 00:35:08 +0300 Subject: [PATCH 19/72] test rules to be used CurrentTestStore in order to prevent incorrect access --- .../asynctest/rule/AtomicValueRule.ceylon | 22 +++---- source/herd/asynctest/rule/CounterRule.ceylon | 24 ++++--- .../herd/asynctest/rule/LockAccessRule.ceylon | 21 +++--- source/herd/asynctest/rule/MeterRule.ceylon | 44 ++++++------- source/herd/asynctest/rule/SignalRule.ceylon | 65 +++++++++---------- source/herd/asynctest/rule/Stat.ceylon | 4 +- .../herd/asynctest/rule/StatisticRule.ceylon | 13 ++-- .../rule/TemporaryDirFileRule.ceylon | 28 ++++---- 8 files changed, 106 insertions(+), 115 deletions(-) diff --git a/source/herd/asynctest/rule/AtomicValueRule.ceylon b/source/herd/asynctest/rule/AtomicValueRule.ceylon index 6cf3166..791a509 100644 --- a/source/herd/asynctest/rule/AtomicValueRule.ceylon +++ b/source/herd/asynctest/rule/AtomicValueRule.ceylon @@ -15,31 +15,29 @@ tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class AtomicValueRule( "Initial value source." Element|Element() initial ) satisfies TestRule { + Atomic extractElement() => if ( is Element() initial ) + then instantiateAtomic( initial() ) else instantiateAtomic( initial ); - Atomic storage = if ( is Element() initial ) - then instantiateAtomic( initial() ) else instantiateAtomic( initial ); + CurrentTestStore> storage = CurrentTestStore>( extractElement ); "An `Element` stored in the rule." - shared Element sense => storage.get(); - assign sense => storage.set( sense ); + shared Element sense => storage.element.get(); + assign sense => storage.element.set( sense ); "Atomically sets the value to the given updated value if the current value == [[expected]]. Returns `false` if current value is not equal to expected one otherwise returns `true`." shared Boolean compareAndSet ( "Value to be compared with current one." Element expected, - "Value to be stored if [[expected]] == current one." Element newValue ) - => storage.compareAndSet( expected, newValue ); + "Value to be stored if [[expected]] == current one." Element newValue + ) => storage.element.compareAndSet( expected, newValue ); "Atomically sets to the given value and returns the old value." - shared Element getAndSet( Element newValue ) => storage.getAndSet( newValue ); + shared Element getAndSet( Element newValue ) => storage.element.getAndSet( newValue ); - shared actual void after( AsyncPrePostContext context ) => context.proceed(); + shared actual void after( AsyncPrePostContext context ) => storage.after( context ); - shared actual void before( AsyncPrePostContext context ) { - sense = if ( is Element() initial ) then initial() else initial; - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => storage.before( context ); } diff --git a/source/herd/asynctest/rule/CounterRule.ceylon b/source/herd/asynctest/rule/CounterRule.ceylon index 394803c..2b6b49c 100644 --- a/source/herd/asynctest/rule/CounterRule.ceylon +++ b/source/herd/asynctest/rule/CounterRule.ceylon @@ -10,31 +10,29 @@ import java.util.concurrent.atomic { tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class CounterRule( "Initial value of the counter." Integer initial = 0 ) satisfies TestRule { - AtomicLong atomicCounter = AtomicLong( initial ); + AtomicLong initialAtomic() => AtomicLong( initial ); + CurrentTestStore atomicCounter = CurrentTestStore( initialAtomic ); "The current value of the counter." - shared Integer counter => atomicCounter.get(); + shared Integer counter => atomicCounter.element.get(); "Adds delta to the current value and returns the result." - shared Integer addAndGet( Integer delta = 1 ) => atomicCounter.addAndGet( delta ); + shared Integer addAndGet( Integer delta = 1 ) => atomicCounter.element.addAndGet( delta ); "Adds delta to the current value and returns the previous value." - shared Integer getAndAdd( Integer delta = 1 ) => atomicCounter.getAndAdd( delta ); + shared Integer getAndAdd( Integer delta = 1 ) => atomicCounter.element.getAndAdd( delta ); "Decrements the value and returns the result." - shared Integer decrementAndGet() => atomicCounter.decrementAndGet(); + shared Integer decrementAndGet() => atomicCounter.element.decrementAndGet(); "Increments the value and returns the result." - shared Integer incrementAndGet() => atomicCounter.incrementAndGet(); + shared Integer incrementAndGet() => atomicCounter.element.incrementAndGet(); "Decrements the value and returns the previous value." - shared Integer getAndDecrement() => atomicCounter.andDecrement; + shared Integer getAndDecrement() => atomicCounter.element.andDecrement; "Increments the value and returns the previous value." - shared Integer getAndIncrement() => atomicCounter.andIncrement; + shared Integer getAndIncrement() => atomicCounter.element.andIncrement; - shared actual void after( AsyncPrePostContext context ) => context.proceed(); + shared actual void after( AsyncPrePostContext context ) => atomicCounter.after( context ); - shared actual void before( AsyncPrePostContext context ) { - atomicCounter.set( initial ); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => atomicCounter.before( context ); } diff --git a/source/herd/asynctest/rule/LockAccessRule.ceylon b/source/herd/asynctest/rule/LockAccessRule.ceylon index 8c79c02..e82b835 100644 --- a/source/herd/asynctest/rule/LockAccessRule.ceylon +++ b/source/herd/asynctest/rule/LockAccessRule.ceylon @@ -52,21 +52,25 @@ shared class LockAccessRule ( unlockCountAtomic.incrementAndGet(); locker.unlock(); } + + string => "lock access rule"; } - variable Box stored = Box( if ( is Element() initial ) then initial() else initial ); + CurrentTestStore stored = CurrentTestStore ( + () => Box( if ( is Element() initial ) then initial() else initial ) + ); "Total number of times the lock has been obtained." - shared Integer lockCount => stored.lockCountAtomic.get(); + shared Integer lockCount => stored.element.lockCountAtomic.get(); "Total number of times the lock has been released." - shared Integer unlockCount => stored.unlockCountAtomic.get(); + shared Integer unlockCount => stored.element.unlockCountAtomic.get(); "Locks the resource and provides access to. Actual resource is stored when `release` is called, but not when `element` is modified." shared class Lock() satisfies Obtainable { - Box box = stored; + Box box = stored.element; variable Boolean locked = true; box.lock(); "Access to the shared resource." @@ -74,8 +78,8 @@ shared class LockAccessRule ( box.unlock(); shared actual void obtain() { - locked = true; box.lock(); + locked = true; } shared actual void release( Throwable? error ) { @@ -88,11 +92,8 @@ shared class LockAccessRule ( } - shared actual void after( AsyncPrePostContext context ) => context.proceed(); + shared actual void after( AsyncPrePostContext context ) => stored.after( context ); - shared actual void before( AsyncPrePostContext context ) { - stored = Box( if ( is Element() initial ) then initial() else initial ); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => stored.before( context ); } diff --git a/source/herd/asynctest/rule/MeterRule.ceylon b/source/herd/asynctest/rule/MeterRule.ceylon index 8eff2ad..4e11ae7 100644 --- a/source/herd/asynctest/rule/MeterRule.ceylon +++ b/source/herd/asynctest/rule/MeterRule.ceylon @@ -28,27 +28,31 @@ by( "Lis" ) since( "0.6.0" ) tagged( "TestRule" ) shared class MeterRule() satisfies TestRule { - "Calculations of the time statistic data." - StatisticCalculator timeCalculator = StatisticCalculator(); + class Box() { + "Calculations of the time statistic data." + shared StatisticCalculator timeCalculator = StatisticCalculator(); + "Calculations of the rate of operations per second statistic data." + shared StatisticCalculator rateCalculator = StatisticCalculator(); + "Time from previous tick." + shared variable Integer previousTime = -1; + + string => "meter rule"; + } - "Calculations of the rate of operations per second statistic data." - StatisticCalculator rateCalculator = StatisticCalculator(); + CurrentTestStore store = CurrentTestStore( Box ); - "Time from previous tick." - variable Integer previousTime = -1; - "Statistic summary for execution time." - shared StatisticSummary timeStatistic => timeCalculator.statisticSummary; + shared StatisticSummary timeStatistic => store.element.timeCalculator.statisticSummary; "Statistic summary for rate (operations per second)." - shared StatisticSummary rateStatistic => rateCalculator.statisticSummary; + shared StatisticSummary rateStatistic => store.element.rateCalculator.statisticSummary; "Starts benchmarking from now and memoizes current system time. To add sample to the statistics data - call [[tick]]." see( `function tick` ) - shared void start() => previousTime = system.nanoseconds; + shared void start() => store.element.previousTime = system.nanoseconds; "Adds clock sample from previous `tick` or from `start`. [[start]] has to be called before the first call of `tick`. @@ -57,23 +61,19 @@ tagged( "TestRule" ) shared class MeterRule() satisfies TestRule see( `function start` ) shared void tick() { "BenchmarkRule: calling `tick` before `start`." - assert ( previousTime >= 0 ); + assert ( store.element.previousTime >= 0 ); + Box box = store.element; Integer now = system.nanoseconds; - Integer delta = now - previousTime; - previousTime = now; - timeCalculator.sample( delta / 1000000.0 ); - rateCalculator.sample( 1000000000.0 / delta ); + Integer delta = now - box.previousTime; + box.previousTime = now; + box.timeCalculator.sample( delta / 1000000.0 ); + box.rateCalculator.sample( 1000000000.0 / delta ); } - shared actual void after( AsyncPrePostContext context ) => context.proceed(); + shared actual void after( AsyncPrePostContext context ) => store.after( context ); - shared actual void before( AsyncPrePostContext context ) { - previousTime = -1; - timeCalculator.reset(); - rateCalculator.reset(); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => store.before( context ); } diff --git a/source/herd/asynctest/rule/SignalRule.ceylon b/source/herd/asynctest/rule/SignalRule.ceylon index 026c543..c10d36a 100644 --- a/source/herd/asynctest/rule/SignalRule.ceylon +++ b/source/herd/asynctest/rule/SignalRule.ceylon @@ -45,38 +45,34 @@ import java.util.concurrent.atomic { tagged( "TestRule" ) since( "0.6.1" ) by( "Lis" ) shared class SignalRule() satisfies TestRule { - variable ReentrantLock lock = ReentrantLock(); - variable Condition condition = lock.newCondition(); + class Box() { + shared ReentrantLock lock = ReentrantLock(); + shared Condition condition = lock.newCondition(); + // counting of await, signal and signal all operations + shared AtomicLong awaitCounts = AtomicLong( 0 ); + shared AtomicLong signalCounts = AtomicLong( 0 ); + shared AtomicLong signalAllCounts = AtomicLong( 0 ); + + string => "signal rule"; + } - // counting of await, signal and signal all operations - AtomicLong awaitCounts = AtomicLong( 0 ); - AtomicLong signalCounts = AtomicLong( 0 ); - AtomicLong signalAllCounts = AtomicLong( 0 ); + CurrentTestStore stored = CurrentTestStore( Box ); "Number of times the `await` has been called. Rested to zero _before each_ test." - shared Integer numberOfAwaits => awaitCounts.get(); + shared Integer numberOfAwaits => stored.element.awaitCounts.get(); "Number of times the `signal` has been called. Rested to zero _before each_ test." - shared Integer numberOfSignal => signalCounts.get(); + shared Integer numberOfSignal => stored.element.signalCounts.get(); "Number of times the `signalAll` has been called. Rested to zero _before each_ test." - shared Integer numberOfSignalAll => signalAllCounts.get(); + shared Integer numberOfSignalAll => stored.element.signalAllCounts.get(); - shared actual void after( AsyncPrePostContext context ) { - context.proceed(); - } + shared actual void after( AsyncPrePostContext context ) => stored.after( context ); - shared actual void before( AsyncPrePostContext context ) { - awaitCounts.set( 0 ); - signalCounts.set( 0 ); - signalAllCounts.set( 0 ); - lock = ReentrantLock(); - condition = lock.newCondition(); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => stored.before( context ); "Causes the current thread to wait until it is signalled or interrupted, or the specified waiting time elapses. @@ -85,40 +81,43 @@ shared class SignalRule() satisfies TestRule { otherwise returns `true`. Note: always returns `true` if `timeMilliseconds` is less or equal to zero. " shared Boolean await( "The maximum time to wait in milliseconds." Integer timeMilliseconds = -1 ) { - lock.lock(); + Box box = stored.element; + box.lock.lock(); try { - awaitCounts.incrementAndGet(); + box.awaitCounts.incrementAndGet(); if ( timeMilliseconds > 0 ) { - return condition.await( timeMilliseconds, TimeUnit.milliseconds ); + return box.condition.await( timeMilliseconds, TimeUnit.milliseconds ); } else { - condition.await(); + box.condition.await(); return true; } } finally { - lock.unlock(); + box.lock.unlock(); } } "Wakes up a one awaiter." shared void signal() { - lock.lock(); + Box box = stored.element; + box.lock.lock(); try { - signalCounts.incrementAndGet(); - condition.signal(); + box.signalCounts.incrementAndGet(); + box.condition.signal(); } - finally { lock.unlock(); } + finally { box.lock.unlock(); } } "Wakes up all awaiters." shared void signalAll() { - lock.lock(); + Box box = stored.element; + box.lock.lock(); try { - signalAllCounts.incrementAndGet(); - condition.signalAll(); + box.signalAllCounts.incrementAndGet(); + box.condition.signalAll(); } - finally { lock.unlock(); } + finally { box.lock.unlock(); } } } diff --git a/source/herd/asynctest/rule/Stat.ceylon b/source/herd/asynctest/rule/Stat.ceylon index 5faf553..338a642 100644 --- a/source/herd/asynctest/rule/Stat.ceylon +++ b/source/herd/asynctest/rule/Stat.ceylon @@ -113,5 +113,7 @@ class StatisticCalculator() { finally { wLock.unlock(); } - } + } + + string => "statistic stream calculation"; } diff --git a/source/herd/asynctest/rule/StatisticRule.ceylon b/source/herd/asynctest/rule/StatisticRule.ceylon index c7dbbc1..30d6d01 100644 --- a/source/herd/asynctest/rule/StatisticRule.ceylon +++ b/source/herd/asynctest/rule/StatisticRule.ceylon @@ -11,23 +11,20 @@ shared class StatisticRule() satisfies TestRule { "Calculations of the statistic data." - StatisticCalculator calculator = StatisticCalculator(); + CurrentTestStore calculator = CurrentTestStore( StatisticCalculator ); "Statistic summary accumulated up to the query moment." see( `function sample` ) - shared StatisticSummary statisticSummary => calculator.statisticSummary; + shared StatisticSummary statisticSummary => calculator.element.statisticSummary; "Thread-safely adds samples to the statistic." see( `value statisticSummary` ) - shared void sample( Float* values ) => calculator.sample( *values ); + shared void sample( Float* values ) => calculator.element.sample( *values ); - shared actual void after( AsyncPrePostContext context ) => context.proceed(); + shared actual void after( AsyncPrePostContext context ) => calculator.after( context ); - shared actual void before( AsyncPrePostContext context ) { - calculator.reset(); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => calculator.before( context ); } diff --git a/source/herd/asynctest/rule/TemporaryDirFileRule.ceylon b/source/herd/asynctest/rule/TemporaryDirFileRule.ceylon index ad3e99c..5057015 100644 --- a/source/herd/asynctest/rule/TemporaryDirFileRule.ceylon +++ b/source/herd/asynctest/rule/TemporaryDirFileRule.ceylon @@ -14,21 +14,19 @@ shared class TemporaryDirectoryRule() satisfies TestRule { - variable Directory.TemporaryDirectory tempDir = temporaryDirectory.TemporaryDirectory( null ); + CurrentTestStore tempDir + = CurrentTestStore( () => temporaryDirectory.TemporaryDirectory( null ) ); "The directory behinds this rule." - shared Directory directory => tempDir; + shared Directory directory => tempDir.element; shared actual void after( AsyncPrePostContext context ) { - tempDir.destroy( null ); - context.proceed(); + tempDir.element.destroy( null ); + tempDir.after( context ); } - shared actual void before( AsyncPrePostContext context ) { - tempDir = temporaryDirectory.TemporaryDirectory( null ); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => tempDir.before( context ); } @@ -39,20 +37,18 @@ shared class TemporaryFileRule() satisfies TestRule { - variable Directory.TemporaryFile tempFile = temporaryDirectory.TemporaryFile( null, null ); + CurrentTestStore tempFile + = CurrentTestStore( () => temporaryDirectory.TemporaryFile( null, null ) ); "The file behinds this rule." - shared File file => tempFile; + shared File file => tempFile.element; shared actual void after( AsyncPrePostContext context ) { - tempFile.destroy( null ); - context.proceed(); + tempFile.element.destroy( null ); + tempFile.after( context ); } - shared actual void before( AsyncPrePostContext context ) { - tempFile = temporaryDirectory.TemporaryFile( null, null ); - context.proceed(); - } + shared actual void before( AsyncPrePostContext context ) => tempFile.before( context ); } From de79081100ff4a8c507e8af3379ae0c107b15265 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 30 Nov 2016 00:35:28 +0300 Subject: [PATCH 20/72] example of ignal rule usage --- .../asynctest/rule/SignalRuleExample.ceylon | 43 +++++++++++++++++++ .../examples/asynctest/rule/package.ceylon | 1 + 2 files changed, 44 insertions(+) create mode 100644 examples/herd/examples/asynctest/rule/SignalRuleExample.ceylon diff --git a/examples/herd/examples/asynctest/rule/SignalRuleExample.ceylon b/examples/herd/examples/asynctest/rule/SignalRuleExample.ceylon new file mode 100644 index 0000000..7b8d15d --- /dev/null +++ b/examples/herd/examples/asynctest/rule/SignalRuleExample.ceylon @@ -0,0 +1,43 @@ +import herd.asynctest { + async, + AsyncTestContext +} +import herd.asynctest.rule { + SignalRule, + testRule +} +import ceylon.test { + test +} +import java.lang { + Thread +} + + +"Provides a simple example of signal rule usage." +async class SignalRuleExample() { + + shared testRule SignalRule rule = SignalRule(); + + test shared void myTest(AsyncTestContext context) { + object thread1 extends Thread() { + shared actual void run() { + print("start await signal"); + rule.await(); // awaits until thread2 signals + print("signal completed"); + context.complete(); + } + } + thread1.start(); + + object thread2 extends Thread() { + shared actual void run() { + sleep(1000); + print("signal!"); + rule.signal(); // wakes up thread1 + } + } + thread2.start(); + } + +} diff --git a/examples/herd/examples/asynctest/rule/package.ceylon b/examples/herd/examples/asynctest/rule/package.ceylon index a599310..1cd4108 100644 --- a/examples/herd/examples/asynctest/rule/package.ceylon +++ b/examples/herd/examples/asynctest/rule/package.ceylon @@ -3,5 +3,6 @@ * [[ContextualRuleExample]] - usage of `herd.asynctest.rule::ContextualRule` which stores values locally to thread. * [[ResourceRuleExample]] - usage of `herd.asynctest.rule::ResourceRule` which provides access to resource stored in module. * [[ServerCustomRule]] - applies custom rule to initialize `ceylon.http.server::Server` and runs it in background. + * [[SignalRuleExample]] - simple example of signal rule usage. " shared package herd.examples.asynctest.rule; From 8862da2b915f2104159e2db1b20b8ac29b3f77bb Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 30 Nov 2016 10:16:47 +0300 Subject: [PATCH 21/72] stringified AtomicFloat --- source/herd/asynctest/internal/Atomic.ceylon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/herd/asynctest/internal/Atomic.ceylon b/source/herd/asynctest/internal/Atomic.ceylon index c85cc6b..45b0c8a 100644 --- a/source/herd/asynctest/internal/Atomic.ceylon +++ b/source/herd/asynctest/internal/Atomic.ceylon @@ -110,7 +110,7 @@ class AtomicFloat( Float initial ) satisfies Atomic shared actual void set( Float v ) => storage.set( Double.doubleToRawLongBits( v ) ); - string => "atomic float of '``get()``'"; + string => "atomic float of '``stringify( get() )``'"; } From ded277e8e5732029f8d5ada37185e059eb2d7ba1 Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 1 Dec 2016 15:13:54 +0300 Subject: [PATCH 22/72] refactor --- .../herd/examples/asynctest/rule/ContextualRuleExample.ceylon | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon b/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon index 0bfed1f..26ef9e8 100644 --- a/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon +++ b/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon @@ -14,7 +14,6 @@ import herd.asynctest.match { } -"Example of contextual rule usage." async class ContextualRuleExample() { shared testRule ContextualRule intValue = ContextualRule(0); From 6be11adcba9c7ed6791dfd035c1e88e567ada552 Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 1 Dec 2016 15:14:28 +0300 Subject: [PATCH 23/72] strigify iterable to show size if it is big --- source/herd/asynctest/internal/stringify.ceylon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/herd/asynctest/internal/stringify.ceylon b/source/herd/asynctest/internal/stringify.ceylon index 00b766d..624db90 100644 --- a/source/herd/asynctest/internal/stringify.ceylon +++ b/source/herd/asynctest/internal/stringify.ceylon @@ -23,7 +23,7 @@ shared String stringify( Anything item ) { else { Integer size = item.size - 1; if ( size > 2 ) { - return "{...}"; + return "{size=``size``}"; } else { StringBuilder builder = StringBuilder(); From e33ed0b8558aa9382d93060ee01973bf9511092d Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 1 Dec 2016 15:15:07 +0300 Subject: [PATCH 24/72] apply arguments annotation to factory and other one-shot functions --- .../herd/asynctest/TestGroupExecutor.ceylon | 5 +- source/herd/asynctest/annotations.ceylon | 65 ++++++------------- .../herd/asynctest/extractSourceValue.ceylon | 26 ++++++++ source/herd/asynctest/module.ceylon | 2 +- source/herd/asynctest/resolveArguments.ceylon | 7 +- 5 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 source/herd/asynctest/extractSourceValue.ceylon diff --git a/source/herd/asynctest/TestGroupExecutor.ceylon b/source/herd/asynctest/TestGroupExecutor.ceylon index ffffd26..53a867b 100644 --- a/source/herd/asynctest/TestGroupExecutor.ceylon +++ b/source/herd/asynctest/TestGroupExecutor.ceylon @@ -88,10 +88,11 @@ class TestGroupExecutor ( else { if ( exists factory = optionalAnnotation( `FactoryAnnotation`, declaration ) ) { // factory exists - use this to instantiate object + value args = resolveArgumentList( factory.factoryFunction, null ); if ( asyncTestRunner.isAsyncFactoryDeclaration( factory.factoryFunction ) ) { return FactoryContext( "``factory.factoryFunction.name``", group ).run ( ( AsyncFactoryContext context ) { - factory.factoryFunction.apply<>().apply( context ); + factory.factoryFunction.apply<>().apply( context, *args ); }, extractTimeout( factory.factoryFunction ) ); @@ -99,7 +100,7 @@ class TestGroupExecutor ( else { return FactoryContext( "``factory.factoryFunction.name``", group ).run ( ( AsyncFactoryContext context ) { - if ( exists ret = factory.factoryFunction.apply<>().apply() ) { + if ( exists ret = factory.factoryFunction.apply<>().apply( *args ) ) { context.fill( ret ); } else { diff --git a/source/herd/asynctest/annotations.ceylon b/source/herd/asynctest/annotations.ceylon index 26c0832..05ddfc8 100644 --- a/source/herd/asynctest/annotations.ceylon +++ b/source/herd/asynctest/annotations.ceylon @@ -15,10 +15,6 @@ import ceylon.test { testExecutor } -import ceylon.language.meta { - - type -} import herd.asynctest.runner { RepeatStrategy, @@ -32,25 +28,8 @@ since( "0.6.0" ) by( "Lis" ) shared annotation TestExecutorAnnotation async() => testExecutor( `class AsyncTestExecutor` ); -"Calls [[source]] to get source value." -since( "0.6.0" ) by( "Lis" ) -Result extractSourceValue( FunctionOrValueDeclaration source, Object? instance ) { - switch ( source ) - case ( is FunctionDeclaration ) { - return if ( !source.toplevel, exists instance ) - then source.memberApply( type( instance ) ).bind( instance )() - else source.apply()(); - } - case ( is ValueDeclaration ) { - return if ( !source.toplevel, exists instance ) - then source.memberApply( type( instance ) ).bind( instance ).get() - else source.apply().get(); - } -} - - "Indicates that test container class or test prepost function have to be instantiated - or called using arguments provided by this annotation, see [[ArgumentsAnnotation.argumentList]]. + or called using arguments provided by this annotation, see [[ArgumentsAnnotation]]. Example: [Hobbit] who => [bilbo]; @@ -66,6 +45,8 @@ Result extractSourceValue( FunctionOrValueDeclaration source, Object? in } } + > Source function may also be marked with `arguments` annotation. + " see( `function arguments` ) since( "0.5.0" ) by( "Lis" ) @@ -75,14 +56,7 @@ shared final annotation class ArgumentsAnnotation ( shared FunctionOrValueDeclaration source ) satisfies OptionalAnnotation -{ - - "Calls [[source]] to get argument stream." - shared Anything[] argumentList ( - "Instance of the test class or `null` if test is performed using top-level function." Object? instance - ) => extractSourceValue( source, instance ); - -} +{} "Provides arguments for a one-shot functions. See [[ArgumentsAnnotation]] for details." @@ -101,11 +75,11 @@ shared annotation ArgumentsAnnotation arguments ( The annotation provides parameterized testing based on collection of test variants. It takes two arguments: - 1. Declaration of function or value which returns a collection of test variants `{TestVariant*}`. + 1. Declaration of function or value which returns a collection of test variants `{TestVariant*}`. 2. Number of failed variants to stop testing. Default is -1 which means no limit. The test will be performed using all test variants returned by the given stream - or while total number of failed variants not exceeds specified limit. + or while total number of failed variants not exceeds specified limit. > [[parameterized]] annotation may occur multiple times at a given test function. > The variants source may be either top-level or tested class shared member. @@ -130,12 +104,12 @@ shared annotation ArgumentsAnnotation arguments ( context.complete(); } - In the above example the function `testIdentity` will be called 3 times: - * testIdentity(context, \"stringIdentity\"); - * testIdentity(context, 1); - * testIdentity(context, 1.0); + In the above example the function `testIdentity` will be called 3 times: + * testIdentity(context, \"stringIdentity\"); + * testIdentity(context, 1); + * testIdentity(context, 1.0); - In order to run test with conventional (non-generic function) type parameters list has to be empty: + In order to run test with conventional (non-generic function) type parameters list has to be empty: [Hobbit] who => [bilbo]; {TestVariant*} dwarves => { TestVariant([], [fili]), @@ -156,7 +130,7 @@ shared annotation ArgumentsAnnotation arguments ( } In this example class `HobbitTester` is instantiated once with argument provided by value `who` and - method `thereAndBackAgain` is called multiply times according to size of `dwarves` stream. + method `thereAndBackAgain` is called multiply times according to size of `dwarves` stream. According to second argument of `parameterized` annotation the test will be stopped if two different invoking of `thereAndBackAgain` with two different arguments report failure. " @@ -200,13 +174,14 @@ shared annotation ParameterizedAnnotation parameterized ( "Indicates that class has to be instantiated using a given factory function. [[factory]] annotation takes declaration of top-level factory function. - Factory function has to take no arguments or take first argument of [[AsyncFactoryContext]] type. - If factory function takes [[AsyncFactoryContext]] as first argument it is executed asynchronously and may - fill the context with instantiated object using [[AsyncFactoryContext.fill]] - or may report on error using [[AsyncFactoryContext.abort]]. Test executor blocks the current thread until - one of [[AsyncFactoryContext.fill]] or [[AsyncFactoryContext.abort]] is called. - Otherwise factory function doesn't take [[AsyncFactoryContext]] as first argument. It is executed synchronously - and has to return instantiated non-optional object or throw an error. + #### Factory function arguments + * The first argument of [[AsyncFactoryContext]] type and the other arguments according to [[arguments]] annotation + (or no more arguments if [[arguments]] annotation is omitted): + The function is executed asynchronously and may fill the context with instantiated object using [[AsyncFactoryContext.fill]] + or may report on error using [[AsyncFactoryContext.abort]]. Test executor blocks the current thread until + one of [[AsyncFactoryContext.fill]] or [[AsyncFactoryContext.abort]] is called. + * If no arguments or function takes arguments according to [[arguments]] annotation: + The function is executed synchronously and has to return instantiated non-optional object or throw an error. #### Example of synchronous instantiation: diff --git a/source/herd/asynctest/extractSourceValue.ceylon b/source/herd/asynctest/extractSourceValue.ceylon new file mode 100644 index 0000000..2307a98 --- /dev/null +++ b/source/herd/asynctest/extractSourceValue.ceylon @@ -0,0 +1,26 @@ +import ceylon.language.meta.declaration { + FunctionOrValueDeclaration, + FunctionDeclaration, + ValueDeclaration +} +import ceylon.language.meta { + type +} + + +"Calls [[source]] to get source value." +since( "0.6.0" ) by( "Lis" ) +Result extractSourceValue( FunctionOrValueDeclaration source, Object? instance ) { + switch ( source ) + case ( is FunctionDeclaration ) { + value args = resolveArgumentList( source, instance ); + return if ( !source.toplevel, exists instance ) + then source.memberApply( type( instance ) ).bind( instance ).apply( *args ) + else source.apply().apply( *args ); + } + case ( is ValueDeclaration ) { + return if ( !source.toplevel, exists instance ) + then source.memberApply( type( instance ) ).bind( instance ).get() + else source.apply().get(); + } +} diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index ef022b8..7d7bb11 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -201,7 +201,7 @@ a given factory function is used to instantiate the class. If no factory function is provided instantiation is done using metamodel staff calling class initializer with arguments - provided with [[arguments]] annotation or without arguments if the annotation is missed. + provided with [[arguments]] annotation or without arguments if the annotation is omitted. > Just a one instance of the test class is used for the overall test runcycle it may cause several misalignments: 1. Test interrelation. Please, remember best-practices say the tests have to be independent. diff --git a/source/herd/asynctest/resolveArguments.ceylon b/source/herd/asynctest/resolveArguments.ceylon index 1adbc4f..da0f1b2 100644 --- a/source/herd/asynctest/resolveArguments.ceylon +++ b/source/herd/asynctest/resolveArguments.ceylon @@ -34,5 +34,10 @@ Anything[] resolveArgumentList ( "Declaration to resolve list" FunctionDeclaration|ClassDeclaration declaration, "Instance of the test class or `null` if not available." Object? instance ) { - return optionalAnnotation( `ArgumentsAnnotation`, declaration )?.argumentList( instance ) else []; + if ( exists argProvider = optionalAnnotation( `ArgumentsAnnotation`, declaration )?.source ) { + return extractSourceValue( argProvider, instance ); + } + else { + return []; + } } From dd9c3a0d622dd6cd2fffc4a19a207c11233197cb Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 1 Dec 2016 18:45:31 +0300 Subject: [PATCH 25/72] applyStatement annotation --- .../herd/asynctest/AsyncTestProcessor.ceylon | 2 +- .../herd/asynctest/TestGroupExecutor.ceylon | 25 +++++++++++-- source/herd/asynctest/rule/package.ceylon | 5 ++- .../herd/asynctest/rule/ruleAnnotation.ceylon | 37 +++++++++++++++++-- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/source/herd/asynctest/AsyncTestProcessor.ceylon b/source/herd/asynctest/AsyncTestProcessor.ceylon index f88ee67..6414344 100644 --- a/source/herd/asynctest/AsyncTestProcessor.ceylon +++ b/source/herd/asynctest/AsyncTestProcessor.ceylon @@ -112,7 +112,7 @@ class AsyncTestProcessor( then extractSourceValue( ann.source, instance ) else repeatOnce; - + "Executes one variant and performs initialization and dispose. Returns output from this variant." VariantTestOutput executeVariant( TestVariant variant ) { diff --git a/source/herd/asynctest/TestGroupExecutor.ceylon b/source/herd/asynctest/TestGroupExecutor.ceylon index 53a867b..9864394 100644 --- a/source/herd/asynctest/TestGroupExecutor.ceylon +++ b/source/herd/asynctest/TestGroupExecutor.ceylon @@ -34,14 +34,16 @@ import ceylon.language.meta.model { import ceylon.language.meta { type, - optionalAnnotation + optionalAnnotation, + sequencedAnnotations } import herd.asynctest.rule { SuiteRule, TestRule, TestRuleAnnotation, - TestStatement + TestStatement, + ApplyStatementAnnotation } import ceylon.test.event { @@ -302,6 +304,19 @@ class TestGroupExecutor ( } + "Returns test statements applied to the given test function only." + TestFunction[] getAppliedStatements( FunctionDeclaration testFunctionDeclaration, Object? instance ) { + value annotations = sequencedAnnotations( `ApplyStatementAnnotation`, testFunctionDeclaration ); + return [ + for ( ann in annotations ) for ( decl in ann.statements ) + TestFunction ( + extractSourceValue( decl, instance ).apply, + extractTimeoutFromObject( decl, "apply" ), decl.name + ) + ]; + } + + "Runs test initializers. Returns `true` if successfull and `false` if errored. if errored fils test report with skipping." Boolean runInitializers( Object? instance, ClassOrInterface<>? instanceType ) { @@ -411,8 +426,10 @@ class TestGroupExecutor ( ExecutionTestOutput[] testReport = [ for ( test in executions ) if ( exists decl = test.functionDeclaration ) AsyncTestProcessor ( - decl, instance, groupContext.childContext( test ), - group, testInitializers, testStatements, testCleaners + decl, instance, groupContext.childContext( test ), group, + testInitializers, + testStatements.append( getAppliedStatements( decl, instance ) ), + testCleaners ).runTest() ]; diff --git a/source/herd/asynctest/rule/package.ceylon b/source/herd/asynctest/rule/package.ceylon index 465e02d..e62b61c 100644 --- a/source/herd/asynctest/rule/package.ceylon +++ b/source/herd/asynctest/rule/package.ceylon @@ -86,7 +86,10 @@ which complete initialization / disposing. Delegation should be used instead of extending. - > Reminding: annotate the rule top-level value or attribute with [[testRule]]! + > Reminding: annotate the rule top-level value or attribute with [[testRule]]! + > Test statement may be applied to a given test function only (and not applied to the rest test functions in the scope): + 1. The statement top-level value or attribute should not be annotated with [[testRule]]. + 2. Test function should be annotated with [[applyStatement]]. " since( "0.6.0" ) by( "Lis" ) diff --git a/source/herd/asynctest/rule/ruleAnnotation.ceylon b/source/herd/asynctest/rule/ruleAnnotation.ceylon index f7d23ff..5f6e1c5 100644 --- a/source/herd/asynctest/rule/ruleAnnotation.ceylon +++ b/source/herd/asynctest/rule/ruleAnnotation.ceylon @@ -1,11 +1,12 @@ import ceylon.language.meta.declaration { - ValueDeclaration + ValueDeclaration, + FunctionDeclaration } "Indicates that the annotated value or attribute identifies a test rule. The value declaration has to satisfy [[SuiteRule]], [[TestRule]] or [[TestStatement]] interfaces." -see( `interface TestRule`, `interface SuiteRule`, `interface TestStatement` ) +see( `interface SuiteRule`, `interface TestRule`, `interface TestStatement` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class TestRuleAnnotation() satisfies OptionalAnnotation @@ -14,6 +15,36 @@ shared final annotation class TestRuleAnnotation() "Indicates that the annotated value or attribute identifies a test rule. The value declaration has to satisfy [[SuiteRule]], [[TestRule]] or [[TestStatement]] interfaces." -see( `interface TestRule`, `interface SuiteRule`, `interface TestStatement` ) +see( `interface SuiteRule`, `interface TestRule`, `interface TestStatement` ) since( "0.6.0" ) by( "Lis" ) shared annotation TestRuleAnnotation testRule() => TestRuleAnnotation(); + + +"Indicates that the given test statements have to be applied to the annotated test function. + Each value declaration from [[statements]] list has to satisfy [[TestStatement]] interface. + The annotation may occur multiple times on the given function. + + > If statement from the given list is marked with [[testRule]] annotation it will be executed twice! + " +see( `interface TestStatement` ) +since( "0.6.1" ) by( "Lis" ) +shared final annotation class ApplyStatementAnnotation ( + "Statement declarations. Each item has to satisfy [[TestStatement]] interface." + shared ValueDeclaration* statements +) + satisfies SequencedAnnotation +{} + + +"Indicates that the given test statements have to be applied to the annotated test function. + Each value declaration from [[statements]] list has to satisfy [[TestStatement]] interface. + The annotation may occur multiple times on the given function. + + > If statement from the given list is marked with [[testRule]] annotation it will be executed twice! + " +see( `interface TestRule`, `interface TestStatement` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation ApplyStatementAnnotation applyStatement ( + "Statement declarations. Each item has to satisfy [[TestStatement]] interface." + ValueDeclaration* statements +) => ApplyStatementAnnotation( *statements ); From 6ba0838f1e8ad791142939625a98addd48b12ead Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 1 Dec 2016 22:20:40 +0300 Subject: [PATCH 26/72] refactor --- .../asynctest/runner/RepeatStrategy.ceylon | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/herd/asynctest/runner/RepeatStrategy.ceylon b/source/herd/asynctest/runner/RepeatStrategy.ceylon index d258d39..4a970d6 100644 --- a/source/herd/asynctest/runner/RepeatStrategy.ceylon +++ b/source/herd/asynctest/runner/RepeatStrategy.ceylon @@ -41,12 +41,12 @@ tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) shared class RepeatUpToSuccessRun( "Number of repeats limit." Integer maxRepeats = 1 ) satisfies RepeatStrategy { - variable Integer? totalRuns = null; + variable Integer totalRuns = 0; shared actual TestVariantResult? completeOrRepeat( TestVariantResult variant ) { - Integer count = ( totalRuns else 1 ) + 1; + Integer count = ( totalRuns > 0 then totalRuns else 1 ) + 1; if ( variant.overallState == TestState.success || count > maxRepeats ) { - totalRuns = null; + totalRuns = 0; return variant; } else { @@ -64,12 +64,12 @@ tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) shared class RepeatUpToFailedRun( "Number of repeats limit." Integer maxRepeats = 1 ) satisfies RepeatStrategy { - variable Integer? totalRuns = null; + variable Integer totalRuns = 0; shared actual TestVariantResult? completeOrRepeat( TestVariantResult variant ) { - Integer count = ( totalRuns else 1 ) + 1; + Integer count = ( totalRuns > 0 then totalRuns else 1 ) + 1; if ( variant.overallState > TestState.success || count > maxRepeats ) { - totalRuns = null; + totalRuns = 0; return variant; } else { @@ -87,12 +87,12 @@ tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) shared class RepeatUpToFailureMessage( "Number of repeats limit." Integer maxRepeats = 1 ) satisfies RepeatStrategy { - variable Integer? totalRuns = null; + variable Integer totalRuns = 0; shared actual TestVariantResult? completeOrRepeat( TestVariantResult variant ) { - Integer count = ( totalRuns else 1 ) + 1; + Integer count = ( totalRuns > 0 then totalRuns else 1 ) + 1; if ( variant.overallState > TestState.success || count > maxRepeats ) { - totalRuns = null; + totalRuns = 0; for ( item in variant.testOutput ) { if ( exists reason = item.error ) { return TestVariantResult ( From f576ab94be5947e28ca20b49034a404ad878cabf Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 1 Dec 2016 23:25:56 +0300 Subject: [PATCH 27/72] add several reports at a time --- source/herd/asynctest/VariantResultBuilder.ceylon | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/herd/asynctest/VariantResultBuilder.ceylon b/source/herd/asynctest/VariantResultBuilder.ceylon index a886459..2b81ff5 100644 --- a/source/herd/asynctest/VariantResultBuilder.ceylon +++ b/source/herd/asynctest/VariantResultBuilder.ceylon @@ -26,10 +26,12 @@ shared class VariantResultBuilder() { } "Adds test output to the variant result." - shared void addOutput( TestOutput output ) { - outs.add( output ); - if ( overallState < output.state ) { - overallState = output.state; + shared void addOutput( TestOutput* outputs ) { + for ( output in outputs ) { + outs.add( output ); + if ( overallState < output.state ) { + overallState = output.state; + } } } From 522c64c01c9bcbcb89675b34a26aff8cb54c2a4c Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 00:52:11 +0300 Subject: [PATCH 28/72] a couple matchers --- .../asynctest/match/iterableMatchers.ceylon | 12 ++++++++ .../herd/asynctest/match/mapMatchers.ceylon | 28 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/source/herd/asynctest/match/iterableMatchers.ceylon b/source/herd/asynctest/match/iterableMatchers.ceylon index 5c48733..615a8ca 100644 --- a/source/herd/asynctest/match/iterableMatchers.ceylon +++ b/source/herd/asynctest/match/iterableMatchers.ceylon @@ -85,6 +85,18 @@ shared class DoesNotContain( "Item to check if matching stream doesn't co } +"Verifies if matching stream of `Sequence` does _not_ contain duplicate elements." +tagged( "Streams" ) since( "0.6.1" ) by( "Lis" ) +shared class ContainsNoDuplicates() + satisfies Matcher<[Value*]> +{ + shared actual MatchResult match( [Value*] val ) + => MatchResult( "stream doesn't contain duplicates", val.distinct.size == val.size ); + + shared actual String string => "stream doesn't contain duplicates"; +} + + "Verifies if matching stream of `Iterable` contains every item from the given `elements` stream." tagged( "Streams" ) since( "0.4.0" ) by( "Lis" ) shared class ContainsEvery( "Elements to check if matching stream contains every item from." {Value&Object*} elements ) diff --git a/source/herd/asynctest/match/mapMatchers.ceylon b/source/herd/asynctest/match/mapMatchers.ceylon index 5339724..d190e04 100644 --- a/source/herd/asynctest/match/mapMatchers.ceylon +++ b/source/herd/asynctest/match/mapMatchers.ceylon @@ -6,7 +6,7 @@ import herd.asynctest.internal { "Verifies if matching `map` defines the given key `key`, see `Map.defines`." tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) -shared class DefinesKey( "Item to check if map defines." Value key ) +shared class DefinesKey( "Key to check if map defines." Value key ) satisfies Matcher> given Value satisfies Object { @@ -17,6 +17,19 @@ shared class DefinesKey( "Item to check if map defines." Value key ) } +"Verifies if matching `map` doesn't define the given key `key`, see `Map.defines`." +tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) +shared class DoesNotDefineKey( "Key to check if map doesn't define." Value key ) + satisfies Matcher> + given Value satisfies Object +{ + shared actual MatchResult match( Map val ) + => MatchResult( "map ``stringify( val )`` doesn't define ``stringify( key )``", !val.defines( key ) ); + + shared actual String string => "map doesn't define ``stringify( key )``"; +} + + "Verifies if matching `map` contains the given item `item`." tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) shared class ContainsItem( "Item to check if map contains." Value item ) @@ -30,6 +43,19 @@ shared class ContainsItem( "Item to check if map contains." Value item ) } +"Verifies if matching `map` doesn't contain the given item `item`." +tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) +shared class DoesNotContainItem( "Item to check if map doesn't contain." Value item ) + satisfies Matcher> + given Value satisfies Object +{ + shared actual MatchResult match( Map val ) + => MatchResult( "map ``stringify( val )`` doesn't contain ``stringify( item )``", !val.items.contains( item ) ); + + shared actual String string => "map doesn't contain ``stringify( item )``"; +} + + "Verifies if matching `map` contains the given item `item` with the given key `key`. Items are compared using operator `==`." tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) From fda8ba3364d94cc00fa810ea15a4fa676c5ba80b Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 00:56:14 +0300 Subject: [PATCH 29/72] config --- .ceylon/.gitignore | 2 -- .ceylon/config | 2 -- .ceylon/config~ | 4 +--- 3 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 .ceylon/.gitignore diff --git a/.ceylon/.gitignore b/.ceylon/.gitignore deleted file mode 100644 index 02fa9c9..0000000 --- a/.ceylon/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/my.format -/my.format~ diff --git a/.ceylon/config b/.ceylon/config index f094ded..c86fa07 100644 --- a/.ceylon/config +++ b/.ceylon/config @@ -7,5 +7,3 @@ source=examples [defaults] encoding=UTF-8 -[formattool] -profile=my diff --git a/.ceylon/config~ b/.ceylon/config~ index 526cf21..c86fa07 100644 --- a/.ceylon/config~ +++ b/.ceylon/config~ @@ -6,6 +6,4 @@ source=examples [defaults] encoding=UTF-8 - -[formattool] -profile=my + From 65a4f47aafa39b27a942b6efc1dfb80fcae66183 Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 20:22:48 +0300 Subject: [PATCH 30/72] see annotation --- source/herd/asynctest/rule/RuleChain.ceylon | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/herd/asynctest/rule/RuleChain.ceylon b/source/herd/asynctest/rule/RuleChain.ceylon index f2aa215..0be3b54 100644 --- a/source/herd/asynctest/rule/RuleChain.ceylon +++ b/source/herd/asynctest/rule/RuleChain.ceylon @@ -8,6 +8,7 @@ import herd.asynctest { Initialization (i.e. `initialize` methods) is performed with iterating of the `rules` in direct order, while diposing (i.e. `dispose` methods) is performed in reverse order. So, the first initialized is the last disposed." +see( `class TestRuleChain`, `class TestStatementChain` ) tagged( "SuiteRule" ) since( "0.6.0" ) by( "Lis" ) shared class SuiteRuleChain ( "A list of the rules to be chained in order they are provided." SuiteRule* rules @@ -30,6 +31,7 @@ shared class SuiteRuleChain ( Initialization (i.e. `before` methods) is performed with iterating of the `rules` in direct order, while diposing (i.e. `after` methods) is performed in reverse order. So, the first initialized is the last disposed." +see( `class SuiteRuleChain`, `class TestStatementChain` ) tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class TestRuleChain ( "A list of the rules to be chained in order they are provided." TestRule* rules @@ -50,6 +52,7 @@ shared class TestRuleChain ( "Applies test statements in the given order." tagged( "TestStatement" ) since( "0.6.1" ) by( "Lis" ) +see( `class SuiteRuleChain`, `class TestRuleChain` ) shared class TestStatementChain ( "A list of the statements to be applied in order they are provided." TestStatement* statements ) satisfies TestStatement { From 270687d1cca5e0cbd260160ef20b34e93ecfc575 Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 20:23:59 +0300 Subject: [PATCH 31/72] lock-free --- source/herd/asynctest/rule/MeterRule.ceylon | 3 +- source/herd/asynctest/rule/Stat.ceylon | 125 +++++++++--------- .../herd/asynctest/rule/StatisticRule.ceylon | 2 +- 3 files changed, 65 insertions(+), 65 deletions(-) diff --git a/source/herd/asynctest/rule/MeterRule.ceylon b/source/herd/asynctest/rule/MeterRule.ceylon index 4e11ae7..0181130 100644 --- a/source/herd/asynctest/rule/MeterRule.ceylon +++ b/source/herd/asynctest/rule/MeterRule.ceylon @@ -3,7 +3,8 @@ import herd.asynctest { } -"Collects statistic data on an execution time and on a rate (per second) at which a set of events occur. +"Lock-free and thread-safely collects statistic data on an execution time and on a rate (per second) + at which a set of events occur. Statistic data is reset before _each_ test. To start recording call [[start]]. To record time delta call [[tick]] which records time delta from `start` or previous `tick` and up to now. diff --git a/source/herd/asynctest/rule/Stat.ceylon b/source/herd/asynctest/rule/Stat.ceylon index 338a642..f5fda7b 100644 --- a/source/herd/asynctest/rule/Stat.ceylon +++ b/source/herd/asynctest/rule/Stat.ceylon @@ -1,10 +1,10 @@ -import java.util.concurrent.locks { - ReentrantReadWriteLock, - Lock -} import java.lang { Math } +import java.util.concurrent.atomic { + AtomicReference +} + "Statistic summary from some variate values stream." see( `class StatisticRule`, `class MeterRule` ) @@ -31,89 +31,88 @@ shared class StatisticSummary ( } -"Calculates statistic data for stream of variate values." -by( "Lis" ) since( "0.6.0" ) -class StatisticCalculator() { - "Statistics locker." - ReentrantReadWriteLock lock = ReentrantReadWriteLock(); - Lock rLock = lock.readLock(); - Lock wLock = lock.writeLock(); - +"Current values of statistic calculations." +by( "Lis" ) since( "0.6.1" ) +class StatisticStream ( "Minimum of the values that have been statisticaly treated." - variable Float min = infinity; + shared Float min = infinity, "Maximum of the values that have been statisticaly treated." - variable Float max = -infinity; + shared Float max = -infinity, "Mean calculated within Welford's method of standard deviation computation." - variable Float mean = 0.0; + shared Float mean = 0.0, "Second moment used in Welford's method of standard deviation computation." - variable Float m2 = 0.0; + shared Float m2 = 0.0, "The number of the values that have been statisticaly treated." - variable Integer size = 0; - + shared Integer size = 0 +) +{ "Returns variance of the values that have been statisticaly treated. The variance is mean((x-mean(x))^2)." - Float variance => if ( size > 1 ) then m2 / ( size - 1 ) else 0.0; + shared Float variance => if ( size > 1 ) then m2 / ( size - 1 ) else 0.0; "Returns standard deviation of the values that have been statisticaly treated. Standard deviation is `variance^0.5`." - Float standardDeviation => Math.sqrt( variance ); + shared Float standardDeviation => Math.sqrt( variance ); + + "Returns new stream with added samples." + shared StatisticStream addSamples( Float* values ) { + variable Float min = this.min; + variable Float max = this.max; + variable Float mean = this.mean; + variable Float m2 = this.m2; + variable Integer size = this.size; + + for ( item in values ) { + size ++; + if ( item < min ) { min = item; } + if ( max < item ) { max = item; } + // Welford's method for mean and variance + Float delta = item - mean; + mean += delta / size; + m2 += delta * ( item - mean ); + } + if ( size < 2 ) { m2 = 0.0; } + + return StatisticStream( min, max, mean, m2, size ); + } + + "Returns summary for this stream." + shared StatisticSummary summary => StatisticSummary( min, max, mean, standardDeviation, size ); +} + + +"Calculates statistic data for stream of variate values." +see( `class StatisticRule`, `class MeterRule` ) +by( "Lis" ) since( "0.6.0" ) +class StatisticCalculator() { - variable StatisticSummary? validStat = null; + AtomicReference stat = AtomicReference( StatisticStream() ); + "Resets calculator to start statistic collecting from scratch." shared void reset() { - wLock.lock(); - try { - validStat = null; - min = infinity; - max = -infinity; - mean = 0.0; - m2 = 0.0; - size = 0; - } - finally { - wLock.unlock(); + variable StatisticStream s = stat.get(); + StatisticStream emptyStat = StatisticStream(); + while ( !stat.compareAndSet( s, emptyStat ) ) { + s = stat.get(); } } "Statistic summary accumulated up to the query moment." see( `function sample` ) - shared StatisticSummary statisticSummary { - rLock.lock(); - try { - if ( exists s = validStat ) { - return s; - } - else { - value s = StatisticSummary( min, max, mean, standardDeviation, size ); - validStat = s; - return s; - } - } - finally { rLock.unlock(); } - } + shared StatisticSummary statisticSummary => stat.get().summary; "Thread-safely adds samples to the statistic." see( `value statisticSummary` ) shared void sample( Float* values ) { - wLock.lock(); - try { - validStat = null; - for ( item in values ) { - size ++; - if ( item < min ) { min = item; } - if ( max < item ) { max = item; } - // Welford's method for mean and variance - Float delta = item - mean; - mean += delta / size; - m2 += delta * ( item - mean ); - } - if ( size < 2 ) { m2 = 0.0; } - } - finally { - wLock.unlock(); + variable StatisticStream sOld = stat.get(); + variable StatisticStream sNew = sOld.addSamples( *values ); + while ( !stat.compareAndSet( sOld, sNew ) ) { + sOld = stat.get(); + sNew = sOld.addSamples( *values ); } } - string => "statistic stream calculation"; + string => "statistic stream calculation"; + } diff --git a/source/herd/asynctest/rule/StatisticRule.ceylon b/source/herd/asynctest/rule/StatisticRule.ceylon index 30d6d01..61c5295 100644 --- a/source/herd/asynctest/rule/StatisticRule.ceylon +++ b/source/herd/asynctest/rule/StatisticRule.ceylon @@ -3,7 +3,7 @@ import herd.asynctest { } -"Provides statistics of some variate values. +"Lock-free and thread-safely collects statistics of some variate values. Statistic data is reseted before _each_ test. " see( `class StatisticSummary` ) tagged( "TestRule" ) by( "Lis" ) since( "0.6.0" ) From ed3ef37f7163ee49ba5b7ae59ae182d12306665a Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 21:30:55 +0300 Subject: [PATCH 32/72] docs --- source/herd/asynctest/annotations.ceylon | 3 +-- source/herd/asynctest/module.ceylon | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/herd/asynctest/annotations.ceylon b/source/herd/asynctest/annotations.ceylon index 05ddfc8..e685793 100644 --- a/source/herd/asynctest/annotations.ceylon +++ b/source/herd/asynctest/annotations.ceylon @@ -213,7 +213,6 @@ see( `function factory`, `interface AsyncFactoryContext` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class FactoryAnnotation ( "Function used to instantiate anotated class. - Has to take no arguments or just a one argument of [[AsyncFactoryContext]] type. Has to return an instance of the looking class." shared FunctionDeclaration factoryFunction ) @@ -226,7 +225,7 @@ see( `interface AsyncFactoryContext` ) since( "0.6.0" ) by( "Lis" ) shared annotation FactoryAnnotation factory ( "Function used to instantiate anotated class. - Has to take no arguments or just a one argument of [[AsyncFactoryContext]] type. + See [[FactoryAnnotation]] for the requirements to the factory function arguments. Has to return an instance of the looking class." FunctionDeclaration factoryFunction diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index 7d7bb11..4e06fbe 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -199,7 +199,7 @@ some complex logic or some asynchronous operations. If the class declaration is marked with [[factory]] annotation a given factory function is used to instantiate the class. - + If no factory function is provided instantiation is done using metamodel staff calling class initializer with arguments provided with [[arguments]] annotation or without arguments if the annotation is omitted. From 97930e4253205d198cf6f8d1536ccce9b010e200 Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 22:50:14 +0300 Subject: [PATCH 33/72] apply only to correct attributes --- source/herd/asynctest/rule/ruleAnnotation.ceylon | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/herd/asynctest/rule/ruleAnnotation.ceylon b/source/herd/asynctest/rule/ruleAnnotation.ceylon index 5f6e1c5..c8ab998 100644 --- a/source/herd/asynctest/rule/ruleAnnotation.ceylon +++ b/source/herd/asynctest/rule/ruleAnnotation.ceylon @@ -2,6 +2,9 @@ import ceylon.language.meta.declaration { ValueDeclaration, FunctionDeclaration } +import ceylon.language.meta.model { + ValueModel +} "Indicates that the annotated value or attribute identifies a test rule. @@ -9,7 +12,8 @@ import ceylon.language.meta.declaration { see( `interface SuiteRule`, `interface TestRule`, `interface TestStatement` ) since( "0.6.0" ) by( "Lis" ) shared final annotation class TestRuleAnnotation() - satisfies OptionalAnnotation + satisfies OptionalAnnotation|ValueModel|ValueModel> {} From f7928618798fb3ba507a09bd6df890d086d41330 Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 2 Dec 2016 22:51:15 +0300 Subject: [PATCH 34/72] instantiate test class using default constructor --- source/herd/asynctest/TestGroupExecutor.ceylon | 8 ++++++-- source/herd/asynctest/exceptions.ceylon | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/source/herd/asynctest/TestGroupExecutor.ceylon b/source/herd/asynctest/TestGroupExecutor.ceylon index 9864394..e61ead0 100644 --- a/source/herd/asynctest/TestGroupExecutor.ceylon +++ b/source/herd/asynctest/TestGroupExecutor.ceylon @@ -113,9 +113,13 @@ class TestGroupExecutor ( ); } } + else if ( exists constructor = declaration.defaultConstructor ) { + // no factory specified - just instantiate using default constructor + return constructor.invoke( [], *resolveArgumentList( declaration, null ) ); + } else { - // no factory specified - just instantiate - return declaration.instantiate( [], *resolveArgumentList( declaration, null ) ); + // no factory and no default constructor - throw exception + throw IncompatibleInstantiation( declaration ); } } } diff --git a/source/herd/asynctest/exceptions.ceylon b/source/herd/asynctest/exceptions.ceylon index a3c365f..f8f85f4 100644 --- a/source/herd/asynctest/exceptions.ceylon +++ b/source/herd/asynctest/exceptions.ceylon @@ -4,6 +4,10 @@ import ceylon.language.meta { import ceylon.test.engine { TestAbortedException } +import ceylon.language.meta.declaration { + ClassDeclaration +} + "Exception which shows that time out has been reached." since( "0.6.0" ) by( "Lis" ) @@ -20,6 +24,17 @@ shared class FactoryReturnsNothing( String factoryTitle ) {} +since( "0.6.1" ) by( "Lis" ) +shared class IncompatibleInstantiation ( + shared ClassDeclaration declaration +) + extends Exception ( + "Unable to instantiate class ``declaration.qualifiedName``. + It has neither default constructor nor factory function." + ) +{} + + "Collects multiple abort reasons in a one exception." since( "0.6.1" ) by( "Lis" ) shared class MultipleAbortException ( From be157dc579cd30b8fa90210b66f4cf97a97eba6e Mon Sep 17 00:00:00 2001 From: Lis Date: Sat, 3 Dec 2016 11:03:55 +0300 Subject: [PATCH 35/72] factory has always to throw when instantiation failed --- source/herd/asynctest/FactoryContext.ceylon | 5 +---- .../herd/asynctest/TestGroupExecutor.ceylon | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/source/herd/asynctest/FactoryContext.ceylon b/source/herd/asynctest/FactoryContext.ceylon index db5a53d..fc3088b 100644 --- a/source/herd/asynctest/FactoryContext.ceylon +++ b/source/herd/asynctest/FactoryContext.ceylon @@ -48,10 +48,7 @@ class FactoryContext ( abortReason = null; instantiatedObject = null; try { factory( this ); } - catch ( Throwable err ) { - instantiatedObject = null; - abortReason = err; - } + catch ( Throwable err ) { abort( err ); } await(); } diff --git a/source/herd/asynctest/TestGroupExecutor.ceylon b/source/herd/asynctest/TestGroupExecutor.ceylon index e61ead0..294600a 100644 --- a/source/herd/asynctest/TestGroupExecutor.ceylon +++ b/source/herd/asynctest/TestGroupExecutor.ceylon @@ -321,9 +321,11 @@ class TestGroupExecutor ( } - "Runs test initializers. Returns `true` if successfull and `false` if errored. - if errored fils test report with skipping." - Boolean runInitializers( Object? instance, ClassOrInterface<>? instanceType ) { + "Runs test initializers. + Returns: + * Empty list if succesfull. + * Nonempty list of test outputs from initializers + cleaners if at least one initializer has been failed." + TestOutput[] runInitializers( Object? instance, ClassOrInterface<>? instanceType ) { // overall test initializaers value testRunInits = if ( exists inst = instance ) @@ -341,11 +343,10 @@ class TestGroupExecutor ( then getAnnotatedPrepost( instance, instanceType ) .append( getSuiteCleaners( instance, instanceType ) ) else getSuiteCleaners( null, null ); - skipGroupTest( initsRet.append( prePostContext.run( cleaners, null ) ) ); - return false; + return initsRet.append( prePostContext.run( cleaners, null ) ); } else { - return true; + return []; } } @@ -417,7 +418,11 @@ class TestGroupExecutor ( Object? instance = instantiate(); ClassOrInterface? instanceType = if ( exists i = instance ) then type( i ) else null; - if ( runInitializers( instance, instanceType ) ) { + if ( nonempty initOuts = runInitializers( instance, instanceType ) ) { + // initializers have been failed - skip test + skipGroupTest( initOuts ); + } + else { // each test run initializers value testInitializers = getAnnotatedPrepost( instance, instanceType ) .append( getTestInitializers( instance, instanceType ) ); From f7dba2cd05eba7ffd4cb83f902bd0e6fd46a34cc Mon Sep 17 00:00:00 2001 From: Lis Date: Sat, 3 Dec 2016 21:18:09 +0300 Subject: [PATCH 36/72] new matchers --- .../herd/asynctest/match/checkMatchers.ceylon | 83 +++++++- .../herd/asynctest/match/listMatchers.ceylon | 191 ++++++++++++++++-- .../asynctest/match/numberMatchers.ceylon | 60 +++++- .../herd/asynctest/match/setMatchers.ceylon | 72 +++++++ 4 files changed, 378 insertions(+), 28 deletions(-) create mode 100644 source/herd/asynctest/match/setMatchers.ceylon diff --git a/source/herd/asynctest/match/checkMatchers.ceylon b/source/herd/asynctest/match/checkMatchers.ceylon index f2022e3..2f853b5 100644 --- a/source/herd/asynctest/match/checkMatchers.ceylon +++ b/source/herd/asynctest/match/checkMatchers.ceylon @@ -4,6 +4,7 @@ import herd.asynctest.internal { typeName } + "Verifies if matching value equals to `merit` using operator `==`." tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) shared class EqualObjects ( @@ -22,6 +23,24 @@ shared class EqualObjects ( } +"Verifies if matching value is _not_ equal to `merit` using operator `!=`." +tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +shared class NotEqualObjects ( + "Value to compare with matching one." Value merit +) + satisfies Matcher + given Value satisfies Object +{ + shared actual MatchResult match( Value val ) + => MatchResult( "``stringify( val )`` != ``stringify( merit )``", val != merit ); + + shared actual String string { + value tVal = `Value`; + return "not equal objects of '``typeName( tVal )``'"; + } +} + + "Verifies if matching value is identical to `merit` using operator `===`." tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) shared class Identical ( @@ -40,7 +59,25 @@ shared class Identical ( } -"The matcher is useful to test classes which implements `eqauls` method. It verifies value equality rules i.e.: +"Verifies if matching value is _not_ identical to `merit` using operator `===`." +tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +shared class NotIdentical ( + "Value to compare with matching one." Value merit +) + satisfies Matcher + given Value satisfies Identifiable +{ + shared actual MatchResult match( Value val ) + => MatchResult( "``stringify( val )`` !== ``stringify( merit )``", !(val === merit) ); + + shared actual String string { + value tVal = `Value`; + return "not identical <``typeName( tVal )``>"; + } +} + + +"The matcher is useful to test classes which implements `equals` method. It verifies value equality rules i.e.: * reflexivity, x==x * symmetry, if x==y then y==x * trasitivity, if x==y and y==z then x==z @@ -105,6 +142,31 @@ shared class EqualWith ( } +"Verifies if matching value is _not_ equal to `merit` using given comparator." +tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +shared class NotEqualWith ( + "Value to compare with matching one." + Value merit, + "Comparator used to compare matching value and merit. + Has to return `true` if values are equal and `false` otherwise." + Boolean comparator( Value first, Value second ) +) + satisfies Matcher + given Value satisfies Identifiable +{ + shared actual MatchResult match( Value val ) + => MatchResult ( + "``stringify( val )`` not equal to ``stringify( merit )`` with comparator", + !comparator( val, merit ) + ); + + shared actual String string { + value tVal = `Value`; + return "not equal with comparator <``typeName( tVal )``>"; + } +} + + "Verifies if matching value is of `Check` type." tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) shared class IsType() @@ -112,7 +174,7 @@ shared class IsType() { shared actual MatchResult match( Anything val ) { value tCheck = `Check`; - return MatchResult( "``stringify( val )`` is ``tCheck``", if ( is Check val ) then true else false ); + return MatchResult( "``stringify( val )`` is ``tCheck``", val is Check ); } shared actual String string { @@ -122,6 +184,23 @@ shared class IsType() } +"Verifies if matching value is _not_ of `Check` type." +tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +shared class IsNotType() + satisfies Matcher +{ + shared actual MatchResult match( Anything val ) { + value tCheck = `Check`; + return MatchResult( "``stringify( val )`` is ``tCheck``", !val is Check ); + } + + shared actual String string { + value tCheck = `Check`; + return "is not <``typeName( tCheck )``>"; + } +} + + "Verifies if matching value is `null`." tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) shared class IsNull() satisfies Matcher diff --git a/source/herd/asynctest/match/listMatchers.ceylon b/source/herd/asynctest/match/listMatchers.ceylon index b093d1c..1e2ece8 100644 --- a/source/herd/asynctest/match/listMatchers.ceylon +++ b/source/herd/asynctest/match/listMatchers.ceylon @@ -5,14 +5,14 @@ import herd.asynctest.internal { } -"Verifies if matching `List` value starts with the given `subList`." +"Verifies if matching `List` value starts with the given [[subList]]." tagged( "Streams", "List" ) since( "0.4.0" ) by( "Lis" ) shared class StartsWith( "Sublist matching value to start with." List subList ) satisfies Matcher> { shared actual MatchResult match( List val ) => MatchResult ( - "list ``stringify( val )`` starts with ``stringify( subList )``", + "``stringify( val )`` starts with ``stringify( subList )``", val.startsWith( subList ) ); @@ -23,14 +23,14 @@ shared class StartsWith( "Sublist matching value to start with." List( "Sublist matching value to end with." List subList ) satisfies Matcher> { shared actual MatchResult match( List val ) => MatchResult ( - "list ``stringify( val )`` ends with ``stringify( subList )``", + "``stringify( val )`` ends with ``stringify( subList )``", val.endsWith( subList ) ); @@ -41,13 +41,13 @@ shared class EndsWith( "Sublist matching value to end with." List } -"Verifies if matching `List` value is beginning point of the given `list`." +"Verifies if matching `List` value is beginning point of the given [[list]]." tagged( "Streams", "List" ) since( "0.4.0" ) by( "Lis" ) shared class Beginning( "List to start with matching value." List list ) satisfies Matcher> { shared actual MatchResult match( List val ) - => MatchResult( "list ``stringify( val )`` is beginning of ``stringify( list )``", list.startsWith( val ) ); + => MatchResult( "``stringify( val )`` is beginning of ``stringify( list )``", list.startsWith( val ) ); shared actual String string { value tVal = `Value`; @@ -56,13 +56,13 @@ shared class Beginning( "List to start with matching value." List } -"Verifies if matching `List` value is finishing point of the given `list`." +"Verifies if matching `List` value is finishing point of the given [[list]]." tagged( "Streams", "List" ) since( "0.4.0" ) by( "Lis" ) -shared class Finishing( List list ) +shared class Finishing( "List which is expected to finish with matching value." List list ) satisfies Matcher> { - shared actual MatchResult match( "List to end with matching value."List val ) - => MatchResult( "list ``stringify( val )`` is finishinig of ``stringify( list )``", list.endsWith( val ) ); + shared actual MatchResult match( List val ) + => MatchResult( "``stringify( val )`` is finishinig of ``stringify( list )``", list.endsWith( val ) ); shared actual String string { value tVal = `Value`; @@ -71,31 +71,182 @@ shared class Finishing( List list ) } -"Verifies if matching `List` value is included in the given `list`." +"Verifies if matching `List` value is included in the given [[list]]. + `SearchableList.includes` is used in order to perform the verification. + " tagged( "Streams", "List" ) since( "0.6.0" ) by( "Lis" ) -shared class Included( SearchableList list, Integer from = 0 ) +shared class Included ( + "List which is expected to include matching value." SearchableList list, + "The smallet index to consider." Integer from = 0 +) satisfies Matcher> { - shared actual MatchResult match( "List to end with matching value."List val ) - => MatchResult( "list ``stringify( val )`` is included in ``stringify( list )``", list.includes( val, from ) ); + shared actual MatchResult match( List val ) + => MatchResult( "``stringify( val )`` is included in ``stringify( list )``", list.includes( val, from ) ); shared actual String string { value tVal = `Value`; - return "included in <``typeName( tVal )``>"; + return "included in SearchableList<``typeName( tVal )``>"; } } -"Verifies if matching `SearchableList` value includes the given `list`." +"Verifies if matching `List` value is included in the given [[list]] at the given [[index]] of the [[list]]. + `SearchableList.includesAt` is used in order to perform the verification. + " +tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +shared class IncludedAt ( + "The index at which the matching value might occur." Integer index, + "List which is expected to include matching value." SearchableList list +) + satisfies Matcher> +{ + shared actual MatchResult match( List val ) + => MatchResult ( + "``stringify( val )`` is included in ``stringify( list )`` at index ``index``", + list.includesAt( index, val ) + ); + + shared actual String string { + value tVal = `Value`; + return "included in SearchableList<``typeName( tVal )``> at index ``index``"; + } +} + + +"Verifies if matching value occurs in the given [[list]] at any index within [[from]]:[[length]] segment. + `SearchableList.occurs` is used in order to perform the verification. + " +tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +shared class Occured ( + "List which is expected to include matching value." SearchableList list, + "The smallet index to consider." Integer from = 0, + "The number of indexes to consider." Integer length = list.size - from +) + satisfies Matcher +{ + shared actual MatchResult match( Value val ) + => MatchResult ( + "``stringify( val )`` occurs in ``stringify( list )`` within ``from``:``length``", + list.occurs( val, from, length ) + ); + + shared actual String string { + value tVal = `Value`; + return "occurs in SearchableList<``typeName( tVal )``> within ``from``:``length``"; + } +} + + +"Verifies if matching value occurs in the given [[list]] at the given [[index]]. + `SearchableList.occursAt` is used in order to perform the verification. + " +tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +shared class OccuredAt ( + "The index at which the matching value might occur." Integer index, + "List which is expected to include matching value at the given [[index]]." SearchableList list +) + satisfies Matcher +{ + shared actual MatchResult match( Value val ) + => MatchResult ( + "``stringify( val )`` occurs in ``stringify( list )`` at index ``index``", + list.occursAt( index, val ) + ); + + shared actual String string { + value tVal = `Value`; + return "occurs in SearchableList<``typeName( tVal )``> at index ``index``"; + } +} + + +"Verifies if matching `SearchableList` value includes the given `list`. + `SearchableList.includes` is used in order to perform the verification. + " tagged( "Streams", "List" ) since( "0.6.0" ) by( "Lis" ) -shared class Includes( List list ) +shared class Includes ( + "List which is expected to be included into matching one." List list, + "The smallet index to consider." Integer from = 0 +) satisfies Matcher> { - shared actual MatchResult match( "List to end with matching value."SearchableList val ) - => MatchResult( "list ``stringify( val )`` includes ``stringify( list )``", val.includes( list ) ); + shared actual MatchResult match( SearchableList val ) + => MatchResult( "``stringify( val )`` includes ``stringify( list )``", val.includes( list, from ) ); + + shared actual String string { + value tVal = `Value`; + return "includes List<``typeName( tVal )``>"; + } +} + + +"Verifies if matching `SearchableList` value includes the given [[list]] at the given [[index]]. + `SearchableList.includesAt` is used in order to perform the verification. + " +tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +shared class IncludesAt ( + "The index at which [[list]] might occur in matching value." Integer index, + "List which is expected to be included into matching one at the given [[index]]." List list +) + satisfies Matcher> +{ + shared actual MatchResult match( SearchableList val ) + => MatchResult ( + "``stringify( val )`` includes ``stringify( list )`` at index ``index``", + val.includesAt( index, list ) + ); + + shared actual String string { + value tVal = `Value`; + return "includes List<``typeName( tVal )``> at index ``index``"; + } +} + + +"Verifies if the given [[element]] occurs in matching `SearchableList` value within `from`:`length` segment. + `SearchableList.occurs` is used in order to perform the verification. + " +tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +shared class Occurs ( + "Value which is expected to occur in the matching one at within `from`:`length` segment." Value element, + "The smallet index to consider." Integer from = 0, + "The number of indexes to consider. If < 0 every element up to the end of matching list is considered." + Integer length = -1 +) + satisfies Matcher> +{ + shared actual MatchResult match( SearchableList val ) + => MatchResult ( + "``stringify( val )`` includes ``stringify( element )``", + val.occurs( element, from, if ( length < 0 ) then val.size - from else length ) + ); + + shared actual String string { + value tVal = `Value`; + return "occurs in List<``typeName( tVal )``>"; + } +} + + +"Verifies if the given [[element]] occurs in matching `SearchableList` value at the given [[index]]. + `SearchableList.occursAt` is used in order to perform the verification. + " +tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +shared class OccursAt ( + "The index at which the given [[element]] might occur in the matching list." Integer index, + "Value which is expected to occur in the matching one at within `from`:`length` segment." Value element +) + satisfies Matcher> +{ + shared actual MatchResult match( SearchableList val ) + => MatchResult ( + "``stringify( element )`` occurs in ``stringify( val )`` at index ``index``", + val.occursAt( index, element ) + ); shared actual String string { value tVal = `Value`; - return "includes <``typeName( tVal )``>"; + return "occurs in List<``typeName( tVal )``> at index ``index``"; } } diff --git a/source/herd/asynctest/match/numberMatchers.ceylon b/source/herd/asynctest/match/numberMatchers.ceylon index 051c2b9..f48fee5 100644 --- a/source/herd/asynctest/match/numberMatchers.ceylon +++ b/source/herd/asynctest/match/numberMatchers.ceylon @@ -3,8 +3,8 @@ import herd.asynctest.internal { } -"Verifies if matching value is negative." -see( `class IsPositive`, `class IsZero`, `class IsNotZero` ) +"Verifies if matching value is negative, i.e. is < 0." +see( `class IsPositive`, `class IsNotPositive`, `class IsNotNegative`, `class IsZero`, `class IsNotZero` ) tagged( "Numbers" ) since( "0.6.0" ) by( "Lis" ) shared class IsNegative() satisfies Matcher @@ -17,8 +17,22 @@ shared class IsNegative() } -"Verifies if matching value is positive." -see( `class IsNegative`, `class IsZero`, `class IsNotZero` ) +"Verifies if matching value is _not_ negative, i.e. is >= 0." +see( `class IsPositive`, `class IsNotPositive`, `class IsNegative`, `class IsZero`, `class IsNotZero` ) +tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +shared class IsNotNegative() + satisfies Matcher + given Value satisfies Number +{ + shared actual MatchResult match( Value val ) + => MatchResult( "``stringify( val )`` is not negative", !val.negative ); + + shared actual String string => "is not negative"; +} + + +"Verifies if matching value is positive, i.e. is > 0." +see( `class IsNegative`, `class IsNotPositive`, `class IsNotNegative`, `class IsZero`, `class IsNotZero` ) tagged( "Numbers" ) since( "0.6.0" ) by( "Lis" ) shared class IsPositive() satisfies Matcher @@ -31,8 +45,22 @@ shared class IsPositive() } +"Verifies if matching value is not positive, i.e. is <= 0." +see( `class IsNegative`, `class IsPositive`, `class IsNotNegative`, `class IsZero`, `class IsNotZero` ) +tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +shared class IsNotPositive() + satisfies Matcher + given Value satisfies Number +{ + shared actual MatchResult match( Value val ) + => MatchResult( "``stringify( val )`` is not positive", !val.positive ); + + shared actual String string => "is not positive"; +} + + "Verifies if matching value is zero." -see( `class IsNegative`, `class IsPositive`, `class IsNotZero` ) +see( `class IsNegative`, `class IsPositive`, `class IsNotPositive`, `class IsNotNegative`, `class IsNotZero` ) tagged( "Numbers" ) since( "0.6.0" ) by( "Lis" ) shared class IsZero() satisfies Matcher @@ -46,7 +74,7 @@ shared class IsZero() "Verifies if matching value is not zero." -see( `class IsNegative`, `class IsPositive`, `class IsZero` ) +see( `class IsNegative`, `class IsPositive`, `class IsNotPositive`, `class IsNotNegative`, `class IsZero` ) tagged( "Numbers" ) since( "0.6.0" ) by( "Lis" ) shared class IsNotZero() satisfies Matcher @@ -111,3 +139,23 @@ shared class IsDefined() } +"Verifies if matching integer is even, i.e. if exists number that i=2*k." +tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +shared class IsEven() satisfies Matcher +{ + shared actual MatchResult match( Integer val ) + => MatchResult( "``stringify( val )`` is even", val.even ); + + shared actual String string => "is even"; +} + + +"Verifies if matching integer is odd, i.e. if exists number that i=2*k+1." +tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +shared class IsOdd() satisfies Matcher +{ + shared actual MatchResult match( Integer val ) + => MatchResult( "``stringify( val )`` is odd", !val.even ); + + shared actual String string => "is odd"; +} diff --git a/source/herd/asynctest/match/setMatchers.ceylon b/source/herd/asynctest/match/setMatchers.ceylon new file mode 100644 index 0000000..452e33b --- /dev/null +++ b/source/herd/asynctest/match/setMatchers.ceylon @@ -0,0 +1,72 @@ +import herd.asynctest.internal { + typeName, + stringify +} + + +"Verifies if matching set is subset of the given [[superset]]." +tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +shared class SubsetOf ( + "Set which is expected to be superset of matching set." Set superset +) + satisfies Matcher> +{ + shared actual MatchResult match( Set val ) + => MatchResult( "``stringify( val )`` is subset of ``stringify( superset )``", val.subset( superset ) ); + + shared actual String string { + value tVal = `Value`; + return "subset of Set<``typeName( tVal )``>"; + } +} + + +"Verifies if matching set is superset of the given [[subset]]." +tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +shared class SupersetOf ( + "Set which is expected to be subset of matching set." Set subset +) + satisfies Matcher> +{ + shared actual MatchResult match( Set val ) + => MatchResult( "``stringify( val )`` is superset of ``stringify( subset )``", val.superset( subset ) ); + + shared actual String string { + value tVal = `Value`; + return "superset of Set<``typeName( tVal )``>"; + } +} + + +"Verifies if the given [[subset]] is subset of matching set." +tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +shared class Subset ( + "Set which is expected to be subset of matching set." Set subset +) + satisfies Matcher> +{ + shared actual MatchResult match( Set val ) + => MatchResult( "``stringify( subset )`` is subset of ``stringify( val )``", subset.subset( val ) ); + + shared actual String string { + value tVal = `Value`; + return "subset of Set<``typeName( tVal )``>"; + } +} + + +"Verifies if the given [[superset]] is superset of matching set." +tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +shared class Superset ( + "Set which is expected to be superset of matching set." Set superset +) + satisfies Matcher> +{ + shared actual MatchResult match( Set val ) + => MatchResult( "``stringify( val )`` is superset of ``stringify( superset )``", superset.superset( val ) ); + + shared actual String string { + value tVal = `Value`; + return "superset of Set<``typeName( tVal )``>"; + } +} From 6e6cd94683774ff4871732914d041338975e0d73 Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 4 Dec 2016 11:02:03 +0300 Subject: [PATCH 37/72] refactor stat calculation --- source/herd/asynctest/rule/MeterRule.ceylon | 16 ++- source/herd/asynctest/rule/Stat.ceylon | 135 +++++++++++++----- .../herd/asynctest/rule/StatisticRule.ceylon | 13 +- 3 files changed, 121 insertions(+), 43 deletions(-) diff --git a/source/herd/asynctest/rule/MeterRule.ceylon b/source/herd/asynctest/rule/MeterRule.ceylon index 0181130..d401c8d 100644 --- a/source/herd/asynctest/rule/MeterRule.ceylon +++ b/source/herd/asynctest/rule/MeterRule.ceylon @@ -1,6 +1,9 @@ import herd.asynctest { AsyncPrePostContext } +import java.util.concurrent.atomic { + AtomicLong +} "Lock-free and thread-safely collects statistic data on an execution time and on a rate (per second) @@ -35,7 +38,7 @@ tagged( "TestRule" ) shared class MeterRule() satisfies TestRule "Calculations of the rate of operations per second statistic data." shared StatisticCalculator rateCalculator = StatisticCalculator(); "Time from previous tick." - shared variable Integer previousTime = -1; + shared AtomicLong previousTime = AtomicLong( -1 ); string => "meter rule"; } @@ -50,10 +53,10 @@ tagged( "TestRule" ) shared class MeterRule() satisfies TestRule shared StatisticSummary rateStatistic => store.element.rateCalculator.statisticSummary; - "Starts benchmarking from now and memoizes current system time. + "Starts metering from now and memoizes current system time. To add sample to the statistics data - call [[tick]]." see( `function tick` ) - shared void start() => store.element.previousTime = system.nanoseconds; + shared void start() => store.element.previousTime.set( system.nanoseconds ); "Adds clock sample from previous `tick` or from `start`. [[start]] has to be called before the first call of `tick`. @@ -61,13 +64,12 @@ tagged( "TestRule" ) shared class MeterRule() satisfies TestRule throws ( `class AssertionError`, "If called before `start`." ) see( `function start` ) shared void tick() { - "BenchmarkRule: calling `tick` before `start`." - assert ( store.element.previousTime >= 0 ); + "MeterRule: calling `tick` before `start`." + assert ( store.element.previousTime.get() >= 0 ); Box box = store.element; Integer now = system.nanoseconds; - Integer delta = now - box.previousTime; - box.previousTime = now; + Integer delta = system.nanoseconds - box.previousTime.getAndSet( now ); box.timeCalculator.sample( delta / 1000000.0 ); box.rateCalculator.sample( 1000000000.0 / delta ); } diff --git a/source/herd/asynctest/rule/Stat.ceylon b/source/herd/asynctest/rule/Stat.ceylon index f5fda7b..7c4bd62 100644 --- a/source/herd/asynctest/rule/Stat.ceylon +++ b/source/herd/asynctest/rule/Stat.ceylon @@ -6,7 +6,7 @@ import java.util.concurrent.atomic { } -"Statistic summary from some variate values stream." +"Statistic summary for a stream of variate values." see( `class StatisticRule`, `class MeterRule` ) by( "Lis" ) since( "0.6.0" ) shared class StatisticSummary ( @@ -33,34 +33,35 @@ shared class StatisticSummary ( "Current values of statistic calculations." by( "Lis" ) since( "0.6.1" ) -class StatisticStream ( +class StatisticStream { + "Minimum of the values that have been statisticaly treated." - shared Float min = infinity, + shared Float min; "Maximum of the values that have been statisticaly treated." - shared Float max = -infinity, + shared Float max; "Mean calculated within Welford's method of standard deviation computation." - shared Float mean = 0.0, + shared Float mean; "Second moment used in Welford's method of standard deviation computation." - shared Float m2 = 0.0, + shared Float m2; "The number of the values that have been statisticaly treated." - shared Integer size = 0 -) -{ - "Returns variance of the values that have been statisticaly treated. - The variance is mean((x-mean(x))^2)." - shared Float variance => if ( size > 1 ) then m2 / ( size - 1 ) else 0.0; - - "Returns standard deviation of the values that have been statisticaly treated. - Standard deviation is `variance^0.5`." - shared Float standardDeviation => Math.sqrt( variance ); + shared Integer size; - "Returns new stream with added samples." - shared StatisticStream addSamples( Float* values ) { - variable Float min = this.min; - variable Float max = this.max; - variable Float mean = this.mean; - variable Float m2 = this.m2; - variable Integer size = this.size; + "New empty stream." + shared new empty() { + min = infinity; + max = -infinity; + mean = 0.0; + m2 = 0.0; + size = 0; + } + + "New stream by the given values." + shared new withValues( Float* values ) { + variable Float min = infinity; + variable Float max = -infinity; + variable Float mean = 0.0; + variable Float m2 = 0.0; + variable Integer size = 0; for ( item in values ) { size ++; @@ -72,8 +73,60 @@ class StatisticStream ( m2 += delta * ( item - mean ); } if ( size < 2 ) { m2 = 0.0; } + + this.min = min; + this.max = max; + this.mean = mean; + this.m2 = m2; + this.size = size; + } + + "New stream by combination of two streams." + shared new combined( StatisticStream first, StatisticStream second ) { + Float min = first.min < second.min then first.min else second.min; + Float max = first.max > second.max then first.max else second.max; + Integer size = first.size + second.size; + Float firstRatio = first.size.float / size; + Float secondSize = second.size.float; + Float secondRatio = secondSize / size; + Float mean = first.mean * firstRatio + second.mean * secondRatio; + Float delta = second.mean - first.mean; + Float m2 = first.m2 + second.m2 + delta * delta * firstRatio * secondSize; - return StatisticStream( min, max, mean, m2, size ); + this.min = min; + this.max = max; + this.mean = mean; + this.m2 = m2; + this.size = size; + } + + "New stream with precalculated data." + shared new withData( Float min, Float max, Float mean, Float m2, Integer size ) { + this.min = min; + this.max = max; + this.mean = mean; + this.m2 = m2; + this.size = size; + } + + + "Returns variance of the values that have been statisticaly treated. + The variance is mean((x-mean(x))^2)." + shared Float variance => if ( size > 1 ) then m2 / ( size - 1 ) else 0.0; + + "Returns standard deviation of the values that have been statisticaly treated. + Standard deviation is `variance^0.5`." + shared Float standardDeviation => Math.sqrt( variance ); + + "New stream with a one value added to this." + shared StatisticStream sample( Float val ) { + Float min = val < this.min then val else this.min; + Float max = val > this.max then val else this.max; + Float delta = val - this.mean; + Integer size = this.size + 1; + Float mean = this.mean + delta / size; + Float m2 = size > 1 then this.m2 + delta * ( val - mean ) else 0.0; + return StatisticStream.withData( min, max, mean, m2, size ); } "Returns summary for this stream." @@ -86,30 +139,48 @@ see( `class StatisticRule`, `class MeterRule` ) by( "Lis" ) since( "0.6.0" ) class StatisticCalculator() { - AtomicReference stat = AtomicReference( StatisticStream() ); + AtomicReference stat = AtomicReference( StatisticStream.empty() ); "Resets calculator to start statistic collecting from scratch." shared void reset() { variable StatisticStream s = stat.get(); - StatisticStream emptyStat = StatisticStream(); + StatisticStream emptyStat = StatisticStream.empty(); while ( !stat.compareAndSet( s, emptyStat ) ) { s = stat.get(); } } "Statistic summary accumulated up to the query moment." - see( `function sample` ) + see( `function sample`, `function samples` ) shared StatisticSummary statisticSummary => stat.get().summary; - "Thread-safely adds samples to the statistic." - see( `value statisticSummary` ) - shared void sample( Float* values ) { + "Thread-safely adds a one sample to the statistic." + see( `value statisticSummary`, `function samples` ) + shared void sample( Float val ) { variable StatisticStream sOld = stat.get(); - variable StatisticStream sNew = sOld.addSamples( *values ); + variable StatisticStream sNew = sOld.sample( val ); while ( !stat.compareAndSet( sOld, sNew ) ) { sOld = stat.get(); - sNew = sOld.addSamples( *values ); + sNew = sOld.sample( val ); + } + } + + "Thread-safely adds samples to the statistic." + see( `value statisticSummary`, `function sample` ) + shared void samples( Float* values ) { + Integer addedSize = values.size; + if ( addedSize == 1, exists val = values.first ) { + sample( val ); + } + else if ( addedSize > 1 ) { + StatisticStream sAdd = StatisticStream.withValues( *values ); + variable StatisticStream sOld = stat.get(); + variable StatisticStream sNew = StatisticStream.combined( sOld, sAdd ); + while ( !stat.compareAndSet( sOld, sNew ) ) { + sOld = stat.get(); + sNew = StatisticStream.combined( sOld, sAdd ); + } } } diff --git a/source/herd/asynctest/rule/StatisticRule.ceylon b/source/herd/asynctest/rule/StatisticRule.ceylon index 61c5295..fb43303 100644 --- a/source/herd/asynctest/rule/StatisticRule.ceylon +++ b/source/herd/asynctest/rule/StatisticRule.ceylon @@ -3,7 +3,8 @@ import herd.asynctest { } -"Lock-free and thread-safely collects statistics of some variate values. +"Lock-free and thread-safely accumulates statistics data of some variate values. + Doesn't collect values, just accumulates statistic data when sample added - see [[sample]] and [[samples]]. Statistic data is reseted before _each_ test. " see( `class StatisticSummary` ) tagged( "TestRule" ) by( "Lis" ) since( "0.6.0" ) @@ -15,12 +16,16 @@ shared class StatisticRule() satisfies TestRule "Statistic summary accumulated up to the query moment." - see( `function sample` ) + see( `function samples`, `function sample` ) shared StatisticSummary statisticSummary => calculator.element.statisticSummary; + "Thread-safely adds a one sample to the statistic." + see( `value statisticSummary`, `function samples` ) + shared void sample( Float val ) => calculator.element.sample( val ); + "Thread-safely adds samples to the statistic." - see( `value statisticSummary` ) - shared void sample( Float* values ) => calculator.element.sample( *values ); + see( `value statisticSummary`, `function sample` ) + shared void samples( Float* values ) => calculator.element.samples( *values ); shared actual void after( AsyncPrePostContext context ) => calculator.after( context ); From 87d1b8005d667fc59a945e9bf0353dc50ef23518 Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 4 Dec 2016 20:58:48 +0300 Subject: [PATCH 38/72] correct name --- source/herd/asynctest/match/iterableMatchers.ceylon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/herd/asynctest/match/iterableMatchers.ceylon b/source/herd/asynctest/match/iterableMatchers.ceylon index 615a8ca..292fd8d 100644 --- a/source/herd/asynctest/match/iterableMatchers.ceylon +++ b/source/herd/asynctest/match/iterableMatchers.ceylon @@ -282,7 +282,7 @@ shared class Contained ( > Note: `Iterable` is lazy and there is no quarantee to get the same stream when it is iterating second time. While `Sequence` is eager and repeatably iterated." tagged( "Streams" ) since( "0.6.0" ) by( "Lis" ) -shared class IsNotContained ( +shared class NotContained ( "Stream to check if it doesn't contain matching value." [Value*] stream ) satisfies Matcher From 4fac35a087cce4b0bedfac97651045a0e3bd2338 Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 4 Dec 2016 21:28:35 +0300 Subject: [PATCH 39/72] doc --- source/herd/asynctest/annotations.ceylon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/herd/asynctest/annotations.ceylon b/source/herd/asynctest/annotations.ceylon index e685793..da93015 100644 --- a/source/herd/asynctest/annotations.ceylon +++ b/source/herd/asynctest/annotations.ceylon @@ -100,14 +100,14 @@ shared annotation ArgumentsAnnotation arguments ( void testIdentity(AsyncTestContext context, Value arg) given Value satisfies Object { - context.assertThat(identity(arg), EqualObjects(arg), \"\", true ); + context.assertThat(identity(arg), EqualObjects(arg), \"\", true); context.complete(); } In the above example the function `testIdentity` will be called 3 times: - * testIdentity(context, \"stringIdentity\"); - * testIdentity(context, 1); - * testIdentity(context, 1.0); + * `testIdentity(context, \"stringIdentity\");` + * `testIdentity(context, 1);` + * `testIdentity(context, 1.0);` In order to run test with conventional (non-generic function) type parameters list has to be empty: [Hobbit] who => [bilbo]; From 6db0e3fd8b5e8844239504ff5840270ced3e0582 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 5 Dec 2016 15:22:19 +0300 Subject: [PATCH 40/72] docs --- .../herd/asynctest/match/checkMatchers.ceylon | 1 - .../asynctest/match/exceptionMatchers.ceylon | 14 +++---- .../herd/asynctest/match/mapMatchers.ceylon | 41 +++++++++++++++---- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/source/herd/asynctest/match/checkMatchers.ceylon b/source/herd/asynctest/match/checkMatchers.ceylon index 2f853b5..de5c3af 100644 --- a/source/herd/asynctest/match/checkMatchers.ceylon +++ b/source/herd/asynctest/match/checkMatchers.ceylon @@ -127,7 +127,6 @@ shared class EqualWith ( Boolean comparator( Value first, Value second ) ) satisfies Matcher - given Value satisfies Identifiable { shared actual MatchResult match( Value val ) => MatchResult ( diff --git a/source/herd/asynctest/match/exceptionMatchers.ceylon b/source/herd/asynctest/match/exceptionMatchers.ceylon index 4064b7d..5ae8892 100644 --- a/source/herd/asynctest/match/exceptionMatchers.ceylon +++ b/source/herd/asynctest/match/exceptionMatchers.ceylon @@ -21,12 +21,12 @@ shared class ExceptionHasType() } -"Verifies if matching exception has message of `messageCondition`." +"Verifies if matching exception has message equaled to `messageCondition`." tagged( "Exceptions" ) since( "0.6.0" ) by( "Lis" ) shared class ExceptionHasMessage( "The expected message." String messageCondition ) satisfies Matcher { shared actual MatchResult match( Throwable val ) { - return MatchResult( "excetion has message \"``val.message``\"", val.message == messageCondition ); + return MatchResult( "exception has expected message of \"``val.message``\"", val.message == messageCondition ); } @@ -36,17 +36,17 @@ shared class ExceptionHasMessage( "The expected message." String messageConditio } -"Verifies if matching exception has no cause." +"Verifies if matching exception doesn't have any cause." tagged( "Exceptions" ) since( "0.6.0" ) by( "Lis" ) shared class ExceptionHasNoCause() satisfies Matcher { shared actual MatchResult match( Throwable val ) { - return MatchResult( "excetion has no cause", !val.cause exists ); + return MatchResult( "exception has no cause", !val.cause exists ); } shared actual String string { - return "excetion has no cause"; + return "exception has no cause"; } } @@ -56,11 +56,11 @@ tagged( "Exceptions" ) since( "0.6.0" ) by( "Lis" ) shared class ExceptionHasCause() satisfies Matcher { shared actual MatchResult match( Throwable val ) { - return MatchResult( "excetion has cause", val.cause exists ); + return MatchResult( "exception has cause", val.cause exists ); } shared actual String string { - return "excetion has cause"; + return "exception has cause"; } } diff --git a/source/herd/asynctest/match/mapMatchers.ceylon b/source/herd/asynctest/match/mapMatchers.ceylon index d190e04..d99830c 100644 --- a/source/herd/asynctest/match/mapMatchers.ceylon +++ b/source/herd/asynctest/match/mapMatchers.ceylon @@ -4,7 +4,8 @@ import herd.asynctest.internal { } -"Verifies if matching `map` defines the given key `key`, see `Map.defines`." +"Verifies if matching map defines the given key [[key]], see `Map.defines`." +see( `class DoesNotContainItem`, `class DoesNotDefineKey`, `class ContainsItem` ) tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) shared class DefinesKey( "Key to check if map defines." Value key ) satisfies Matcher> @@ -17,7 +18,8 @@ shared class DefinesKey( "Key to check if map defines." Value key ) } -"Verifies if matching `map` doesn't define the given key `key`, see `Map.defines`." +"Verifies if matching map doesn't define the given key [[key]], see `Map.defines`." +see( `class DoesNotContainItem`, `class ContainsItem`, `class DefinesKey` ) tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) shared class DoesNotDefineKey( "Key to check if map doesn't define." Value key ) satisfies Matcher> @@ -30,7 +32,8 @@ shared class DoesNotDefineKey( "Key to check if map doesn't define." Valu } -"Verifies if matching `map` contains the given item `item`." +"Verifies if matching map contains the given item [[item]]." +see( `class DoesNotContainItem`, `class DoesNotDefineKey`, `class DefinesKey` ) tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) shared class ContainsItem( "Item to check if map contains." Value item ) satisfies Matcher> @@ -43,7 +46,8 @@ shared class ContainsItem( "Item to check if map contains." Value item ) } -"Verifies if matching `map` doesn't contain the given item `item`." +"Verifies if matching map doesn't contain the given item [[item]]." +see( `class ContainsItem`, `class DoesNotDefineKey`, `class DefinesKey` ) tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) shared class DoesNotContainItem( "Item to check if map doesn't contain." Value item ) satisfies Matcher> @@ -56,12 +60,13 @@ shared class DoesNotContainItem( "Item to check if map doesn't contain." } -"Verifies if matching `map` contains the given item `item` with the given key `key`. +"Verifies if matching map contains the given item `[[item]] with the given key [[key]]. Items are compared using operator `==`." +see( `class ItemAtKey` ) tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) shared class ItemByKey ( - "Key to look `item` at." Object key, - "Item to check if map contains under given `key`." Value item + "Key to look [[item]] at." Object key, + "Item to check if map contains under the given [[key]]." Value item ) satisfies Matcher> given Value satisfies Object @@ -73,9 +78,27 @@ shared class ItemByKey ( } -"Verifies if matching value is a key in the given `map`, see `Map.defines`." +"Verifies if the given [[map]] contains the given item [[item]] at the matching value as key. + Items are compared using operator `==`." +see( `class ItemByKey` ) +tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) +shared class ItemAtKey ( + "Map which is expected to contain [[item]] under matching key." Map map, + "Item to check if [[map]] contains under matching key." Value item +) + satisfies Matcher + given Value satisfies Object +{ + shared actual MatchResult match( Value val ) + => MatchResult( string, map.get( val )?.equals( item ) else false ); + + shared actual String string => "matching key referes to item ``stringify( item )`` in the given map"; +} + + +"Verifies if matching value is a key in the given [[map]], see `Map.defines`." tagged( "Streams", "Maps" ) since( "0.4.0" ) by( "Lis" ) -shared class HasKey( "Item to check if map defines." Map map ) +shared class HasKey( "Map which is expected to define matching key." Map map ) satisfies Matcher given Value satisfies Object { From 636741452dbd9a1fc9e624c743634a72954656b8 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 5 Dec 2016 18:58:44 +0300 Subject: [PATCH 41/72] deprecate equal with <=> --- .../generics/typeParameterized.ceylon | 5 +- .../rule/ContextualRuleExample.ceylon | 48 +++++++++---------- .../asynctest/rule/ResourceRuleExample.ceylon | 6 +-- .../asynctest/rule/ServerCustomRule.ceylon | 4 +- .../herd/asynctest/match/checkMatchers.ceylon | 24 +++++----- .../asynctest/match/comparableMatchers.ceylon | 36 -------------- .../asynctest/match/exceptionMatchers.ceylon | 1 + 7 files changed, 44 insertions(+), 80 deletions(-) diff --git a/examples/herd/examples/asynctest/generics/typeParameterized.ceylon b/examples/herd/examples/asynctest/generics/typeParameterized.ceylon index d77287e..4d723a8 100644 --- a/examples/herd/examples/asynctest/generics/typeParameterized.ceylon +++ b/examples/herd/examples/asynctest/generics/typeParameterized.ceylon @@ -7,7 +7,6 @@ import herd.asynctest { TestVariant } import herd.asynctest.match { - EqualObjects, EqualTo } @@ -26,7 +25,7 @@ shared test parameterized(`value identityArgs`) void testIdentity(AsyncTestContext context, Value arg) given Value satisfies Object { - context.assertThat(identity(arg), EqualObjects(arg), "", true ); + context.assertThat(identity(arg), EqualTo(arg), "", true ); context.complete(); } @@ -72,6 +71,6 @@ parameterized(`value sortArgsString`) void testSort(AsyncTestContext context, Element[] stream, Element[] merit) given Element satisfies Comparable { - context.assertThat(sort(stream), EqualObjects(merit), "", true ); + context.assertThat(sort(stream), EqualTo(merit), "", true ); context.complete(); } diff --git a/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon b/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon index 26ef9e8..ed4feb1 100644 --- a/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon +++ b/examples/herd/examples/asynctest/rule/ContextualRuleExample.ceylon @@ -10,7 +10,7 @@ import ceylon.test { test } import herd.asynctest.match { - EqualObjects + EqualTo } @@ -20,60 +20,60 @@ async class ContextualRuleExample() { shared test void contextual(AsyncTestContext context) { - context.assertThat(intValue.get, EqualObjects(0), "initial", true); + context.assertThat(intValue.get, EqualTo(0), "initial", true); try (u1 = intValue.Using(2)) { - context.assertThat(intValue.get, EqualObjects(2), "step", true); - context.assertThat(intValue.get, EqualObjects(2), "rollback", true); + context.assertThat(intValue.get, EqualTo(2), "step", true); + context.assertThat(intValue.get, EqualTo(2), "rollback", true); } - context.assertThat(intValue.get, EqualObjects(0), "second rollback", true); + context.assertThat(intValue.get, EqualTo(0), "second rollback", true); context.complete(); } shared test void complex(AsyncTestContext context) { - context.assertThat(intValue.get, EqualObjects(0), "initial", true); + context.assertThat(intValue.get, EqualTo(0), "initial", true); try (u1 = intValue.Using(2)) { - context.assertThat(intValue.get, EqualObjects(2), "first step", true); + context.assertThat(intValue.get, EqualTo(2), "first step", true); try(u2 = intValue.Using(3)) { - context.assertThat(intValue.get, EqualObjects(3), "second step", true); + context.assertThat(intValue.get, EqualTo(3), "second step", true); } - context.assertThat(intValue.get, EqualObjects(2), "first rollback", true); + context.assertThat(intValue.get, EqualTo(2), "first rollback", true); } - context.assertThat(intValue.get, EqualObjects(0), "second rollback", true); + context.assertThat(intValue.get, EqualTo(0), "second rollback", true); context.complete(); } shared test void same(AsyncTestContext context) { - context.assertThat(intValue.get, EqualObjects(0), "initial", true); + context.assertThat(intValue.get, EqualTo(0), "initial", true); try (u1 = intValue.Using(2)) { - context.assertThat(intValue.get, EqualObjects(2), "first step", true); + context.assertThat(intValue.get, EqualTo(2), "first step", true); try (u1) { - context.assertThat(intValue.get, EqualObjects(2), "second step", true); + context.assertThat(intValue.get, EqualTo(2), "second step", true); } - context.assertThat(intValue.get, EqualObjects(2), "first rollback", true); + context.assertThat(intValue.get, EqualTo(2), "first rollback", true); } - context.assertThat(intValue.get, EqualObjects(0), "second rollback", true); + context.assertThat(intValue.get, EqualTo(0), "second rollback", true); context.complete(); } shared test void manual(AsyncTestContext context) { - context.assertThat(intValue.get, EqualObjects(0), "initial", true); + context.assertThat(intValue.get, EqualTo(0), "initial", true); try (u1 = intValue.Using(2)) { - context.assertThat(intValue.get, EqualObjects(2), "step", true); + context.assertThat(intValue.get, EqualTo(2), "step", true); value u2 = intValue.Using(3); u2.obtain(); - context.assertThat(intValue.get, EqualObjects(3), "step", true); + context.assertThat(intValue.get, EqualTo(3), "step", true); try (u3 = intValue.Using(4)) { - context.assertThat(intValue.get, EqualObjects(4), "step", true); + context.assertThat(intValue.get, EqualTo(4), "step", true); u2.release(null); - context.assertThat(intValue.get, EqualObjects(3), "rollback", true); + context.assertThat(intValue.get, EqualTo(3), "rollback", true); u1.obtain(); - context.assertThat(intValue.get, EqualObjects(2), "step", true); + context.assertThat(intValue.get, EqualTo(2), "step", true); u1.release(null); - context.assertThat(intValue.get, EqualObjects(3), "rollback", true); + context.assertThat(intValue.get, EqualTo(3), "rollback", true); } - context.assertThat(intValue.get, EqualObjects(2), "rollback", true); + context.assertThat(intValue.get, EqualTo(2), "rollback", true); } - context.assertThat(intValue.get, EqualObjects(0), "rollback", true); + context.assertThat(intValue.get, EqualTo(0), "rollback", true); context.complete(); } diff --git a/examples/herd/examples/asynctest/rule/ResourceRuleExample.ceylon b/examples/herd/examples/asynctest/rule/ResourceRuleExample.ceylon index 7d5fe4b..9cef8d8 100644 --- a/examples/herd/examples/asynctest/rule/ResourceRuleExample.ceylon +++ b/examples/herd/examples/asynctest/rule/ResourceRuleExample.ceylon @@ -11,7 +11,7 @@ import ceylon.test { test } import herd.asynctest.match { - EqualObjects + EqualTo } @@ -26,13 +26,13 @@ class ResourceRuleExample( String fileName, String fileContent ) { "Verifies that `resource` name is equal to `fileName`." shared test async void fileNameTest( AsyncTestContext context ) { - context.assertThat( resource.name, EqualObjects( fileName ), "", true ); + context.assertThat( resource.name, EqualTo( fileName ), "", true ); context.complete(); } "Verifies that `resource` content is equal to `fileContent`." shared test async void contentTest( AsyncTestContext context ) { - context.assertThat( resource.textContent(), EqualObjects( fileContent ), "", true ); + context.assertThat( resource.textContent(), EqualTo( fileContent ), "", true ); context.complete(); } diff --git a/examples/herd/examples/asynctest/rule/ServerCustomRule.ceylon b/examples/herd/examples/asynctest/rule/ServerCustomRule.ceylon index 72d0f5c..c04b3f9 100644 --- a/examples/herd/examples/asynctest/rule/ServerCustomRule.ceylon +++ b/examples/herd/examples/asynctest/rule/ServerCustomRule.ceylon @@ -39,7 +39,7 @@ import ceylon.uri { parse } import herd.asynctest.match { - EqualObjects + EqualTo } @@ -104,7 +104,7 @@ class ServerCustomRule ( Integer start = system.milliseconds; value content = req.execute().contents; Integer end = system.milliseconds; - context.assertThat(content, EqualObjects(responseString), "", true); + context.assertThat(content, EqualTo(responseString), "", true); context.succeed("request-response has been taking ``end-start``ms"); context.complete(); } diff --git a/source/herd/asynctest/match/checkMatchers.ceylon b/source/herd/asynctest/match/checkMatchers.ceylon index de5c3af..de3d822 100644 --- a/source/herd/asynctest/match/checkMatchers.ceylon +++ b/source/herd/asynctest/match/checkMatchers.ceylon @@ -6,9 +6,9 @@ import herd.asynctest.internal { "Verifies if matching value equals to `merit` using operator `==`." -tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) -shared class EqualObjects ( - "Value to compare with matching one." Value merit +tagged( "Checkers", "Comparators" ) since( "0.4.0" ) by( "Lis" ) +shared class EqualTo ( + "Expected value." Value merit ) satisfies Matcher given Value satisfies Object @@ -24,9 +24,9 @@ shared class EqualObjects ( "Verifies if matching value is _not_ equal to `merit` using operator `!=`." -tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) -shared class NotEqualObjects ( - "Value to compare with matching one." Value merit +tagged( "Checkers", "Comparators" ) since( "0.6.1" ) by( "Lis" ) +shared class NotEqualTo ( + "Value which is expected to be not equal to matching one." Value merit ) satisfies Matcher given Value satisfies Object @@ -44,7 +44,7 @@ shared class NotEqualObjects ( "Verifies if matching value is identical to `merit` using operator `===`." tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) shared class Identical ( - "Value to compare with matching one." Value merit + "Value which is expected to be identical to matching one." Value merit ) satisfies Matcher given Value satisfies Identifiable @@ -62,7 +62,7 @@ shared class Identical ( "Verifies if matching value is _not_ identical to `merit` using operator `===`." tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) shared class NotIdentical ( - "Value to compare with matching one." Value merit + "Value which is expected to be _not_ identical to matching one." Value merit ) satisfies Matcher given Value satisfies Identifiable @@ -117,10 +117,10 @@ shared class ValueEquality ( } -"Verifies if matching value is equal to `merit` using given comparator." +"Verifies if matching value is equal to [[merit]] using given [[comparator]]." tagged( "Checkers" ) since( "0.4.0" ) by( "Lis" ) shared class EqualWith ( - "Value to compare with matching one." + "Value which is expected to be equal to matching one with the given [[comparator]]." Value merit, "Comparator used to compare matching value and merit. Has to return `true` if values are equal and `false` otherwise." @@ -141,10 +141,10 @@ shared class EqualWith ( } -"Verifies if matching value is _not_ equal to `merit` using given comparator." +"Verifies if matching value is _not_ equal to [[merit]] using given [[comparator]]." tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) shared class NotEqualWith ( - "Value to compare with matching one." + "Value which is expected to be _not_ equal to matching one with the given [[comparator]]." Value merit, "Comparator used to compare matching value and merit. Has to return `true` if values are equal and `false` otherwise." diff --git a/source/herd/asynctest/match/comparableMatchers.ceylon b/source/herd/asynctest/match/comparableMatchers.ceylon index cce6693..ba3ca3a 100644 --- a/source/herd/asynctest/match/comparableMatchers.ceylon +++ b/source/herd/asynctest/match/comparableMatchers.ceylon @@ -169,42 +169,6 @@ shared class NotInRange ( } -"Verifies if matching value is equal to `merit`." -tagged( "Comparators" ) since( "0.4.0" ) by( "Lis" ) -shared class EqualTo ( - "Value to compare with matching one." Value merit -) - satisfies Matcher - given Value satisfies Comparable -{ - shared actual MatchResult match( Value val ) - => MatchResult( "``stringify( val )`` == ``stringify( merit )``", ( val <=> merit ) == equal ); - - shared actual String string { - value tVal = `Value`; - return "equal <``typeName( tVal )``>"; - } -} - - -"Verifies if matching value is not equal to `merit`." -tagged( "Comparators" ) since( "0.4.0" ) by( "Lis" ) -shared class NotEqualTo ( - "Value to compare with matching one." Value merit -) - satisfies Matcher - given Value satisfies Comparable -{ - shared actual MatchResult match( Value val ) - => MatchResult( "``stringify( val )`` != ``stringify( merit )``", ( val <=> merit ) != equal ); - - shared actual String string { - value tVal = `Value`; - return "not equal <``typeName( tVal )``>"; - } -} - - "Verifies if matching value is close to `merit` with the given `tolerance`." tagged( "Comparators" ) since( "0.4.0" ) by( "Lis" ) shared class CloseTo ( diff --git a/source/herd/asynctest/match/exceptionMatchers.ceylon b/source/herd/asynctest/match/exceptionMatchers.ceylon index 5ae8892..8a93ed1 100644 --- a/source/herd/asynctest/match/exceptionMatchers.ceylon +++ b/source/herd/asynctest/match/exceptionMatchers.ceylon @@ -2,6 +2,7 @@ import herd.asynctest.internal { typeName } + "Verifies if matching exception has type of `ExceptionType`." tagged( "Exceptions" ) since( "0.6.0" ) by( "Lis" ) shared class ExceptionHasType() From 170c03a79a22136eeebf3b2de404b65f76d85085 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 5 Dec 2016 21:58:40 +0300 Subject: [PATCH 42/72] docs --- .../herd/asynctest/runner/RepeatRunner.ceylon | 19 +++++++++++-------- .../asynctest/runner/RepeatStrategy.ceylon | 10 +++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/source/herd/asynctest/runner/RepeatRunner.ceylon b/source/herd/asynctest/runner/RepeatRunner.ceylon index 4fea8f8..408cd4d 100644 --- a/source/herd/asynctest/runner/RepeatRunner.ceylon +++ b/source/herd/asynctest/runner/RepeatRunner.ceylon @@ -61,25 +61,27 @@ shared abstract class RepeatRunner() } -"Runner which repeats up to the first successfull run but no more then `maxRepeats`." -see( `class RepeatUpToSuccessRun` ) +"Runner which repeats up to the first successfull run but no more then `maxRepeats`. + Reports result from the latest run." +see( `class RepeatUpToSuccessfulRun` ) tagged( "Runner", "Repeat" ) since( "0.6.0" ) by( "Lis" ) -shared class UpToSuccessRepeater( "Number of repeats limit." Integer maxRepeats = 1 ) +shared class UpToSuccessRepeater( "Maximum number of repeats." Integer maxRepeats = 1 ) extends RepeatRunner() { shared actual class Repeater() extends super.Repeater() { - RepeatStrategy strategy = RepeatUpToSuccessRun( maxRepeats ); + RepeatStrategy strategy = RepeatUpToSuccessfulRun( maxRepeats ); shared actual TestVariantResult? completeOrRepeat( TestVariantResult variant ) => strategy.completeOrRepeat( variant ); } } -"Runner which repeats up to the first failed run but no more then `maxRepeats`." +"Runner which repeats up to the first failed run but no more then `maxRepeats`. + Reports result from the latest run." see( `class RepeatUpToFailedRun` ) tagged( "Runner", "Repeat" ) since( "0.6.0" ) by( "Lis" ) -shared class UpToFailureRepeater( "Number of repeats limit." Integer maxRepeats = 1 ) +shared class UpToFailureRepeater( "Maximum number of repeats." Integer maxRepeats = 1 ) extends RepeatRunner() { shared actual class Repeater() extends super.Repeater() { @@ -89,11 +91,12 @@ shared class UpToFailureRepeater( "Number of repeats limit." Integer maxRepeats } -"Runner which repeats up to the first failure message but no more then `maxRepeats`." +"Runner which repeats up to the first failure message but no more then `maxRepeats`. + Reports the first failure message only" see( `class RepeatUpToFailureMessage` ) tagged( "Runner", "Repeat" ) since( "0.6.0" ) by( "Lis" ) -shared class UpToFailureMessageRepeater( "Number of repeats limit." Integer maxRepeats = 1 ) +shared class UpToFailureMessageRepeater( "Maximum number of repeats." Integer maxRepeats = 1 ) extends RepeatRunner() { shared actual class Repeater() extends super.Repeater() { diff --git a/source/herd/asynctest/runner/RepeatStrategy.ceylon b/source/herd/asynctest/runner/RepeatStrategy.ceylon index 4a970d6..360053e 100644 --- a/source/herd/asynctest/runner/RepeatStrategy.ceylon +++ b/source/herd/asynctest/runner/RepeatStrategy.ceylon @@ -35,11 +35,11 @@ shared object repeatOnce satisfies RepeatStrategy { } -"Repeats up to the first successfull run but no more than `maxRepeats` times. +"Repeats up to the first successful run but no more than `maxRepeats` times. Reports result from the latest run." tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) -shared class RepeatUpToSuccessRun( "Number of repeats limit." Integer maxRepeats = 1 ) satisfies RepeatStrategy { +shared class RepeatUpToSuccessfulRun( "Maximum number of repeats." Integer maxRepeats = 1 ) satisfies RepeatStrategy { variable Integer totalRuns = 0; @@ -62,7 +62,7 @@ shared class RepeatUpToSuccessRun( "Number of repeats limit." Integer maxRepeats Reports result from the latest run." tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) -shared class RepeatUpToFailedRun( "Number of repeats limit." Integer maxRepeats = 1 ) satisfies RepeatStrategy { +shared class RepeatUpToFailedRun( "Maximum number of repeats." Integer maxRepeats = 1 ) satisfies RepeatStrategy { variable Integer totalRuns = 0; @@ -82,10 +82,10 @@ shared class RepeatUpToFailedRun( "Number of repeats limit." Integer maxRepeats "Repeats up to the first failure message but no more than `maxRepeats` times. - Reports just this failed message." + Reports the first failure message only." tagged( "Repeat" ) since( "0.6.0" ) by( "Lis" ) -shared class RepeatUpToFailureMessage( "Number of repeats limit." Integer maxRepeats = 1 ) satisfies RepeatStrategy { +shared class RepeatUpToFailureMessage( "Maximum number of repeats." Integer maxRepeats = 1 ) satisfies RepeatStrategy { variable Integer totalRuns = 0; From 290143cc7bf28c96f551cfc629f6824bf8d19570 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 5 Dec 2016 22:00:18 +0300 Subject: [PATCH 43/72] docs --- source/herd/asynctest/runner/package.ceylon | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/herd/asynctest/runner/package.ceylon b/source/herd/asynctest/runner/package.ceylon index be5aac5..251db41 100644 --- a/source/herd/asynctest/runner/package.ceylon +++ b/source/herd/asynctest/runner/package.ceylon @@ -19,7 +19,7 @@ Example: class MyRunner() satisfies AsyncTestRunner { shared actual void run(AsyncMessageContext context, void testing(AsyncMessageContext context), TestInfo info) { - teting(context); + testing(context); } } @@ -47,8 +47,8 @@ calling `AsyncMessageContext.complete` always leads to test completion and runner has no way to avoid the test completion. Reasons: - 1. Test may be interrupted by timeout. - 2. Synchronized context method calling. + 1. Test may be interrupted by timeout. + 2. Synchronized calling of context methods. " since( "0.6.0" ) by( "Lis" ) From 7bcfc506875ffc7bbecb255394b89c2cb379e307 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 7 Dec 2016 00:30:11 +0300 Subject: [PATCH 44/72] docs --- source/herd/asynctest/rule/RuleChain.ceylon | 12 +++++++++--- source/herd/asynctest/rule/Verifier.ceylon | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/herd/asynctest/rule/RuleChain.ceylon b/source/herd/asynctest/rule/RuleChain.ceylon index 0be3b54..505535e 100644 --- a/source/herd/asynctest/rule/RuleChain.ceylon +++ b/source/herd/asynctest/rule/RuleChain.ceylon @@ -7,7 +7,9 @@ import herd.asynctest { "Applies suite rules in the given order. Initialization (i.e. `initialize` methods) is performed with iterating of the `rules` in direct order, while diposing (i.e. `dispose` methods) is performed in reverse order. So, the first initialized is - the last disposed." + the last disposed. + + > If submited rule is marked with [[testRule]] annotation it will be executed twice." see( `class TestRuleChain`, `class TestStatementChain` ) tagged( "SuiteRule" ) since( "0.6.0" ) by( "Lis" ) shared class SuiteRuleChain ( @@ -30,7 +32,9 @@ shared class SuiteRuleChain ( "Applies test rules in the given order. Initialization (i.e. `before` methods) is performed with iterating of the `rules` in direct order, while diposing (i.e. `after` methods) is performed in reverse order. So, the first initialized is - the last disposed." + the last disposed. + + > If submited rule is marked with [[testRule]] annotation it will be executed twice." see( `class SuiteRuleChain`, `class TestStatementChain` ) tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class TestRuleChain ( @@ -50,7 +54,9 @@ shared class TestRuleChain ( } -"Applies test statements in the given order." +"Applies test statements in the given order. + + > If submited rule is marked with [[testRule]] annotation it will be executed twice." tagged( "TestStatement" ) since( "0.6.1" ) by( "Lis" ) see( `class SuiteRuleChain`, `class TestRuleChain` ) shared class TestStatementChain ( diff --git a/source/herd/asynctest/rule/Verifier.ceylon b/source/herd/asynctest/rule/Verifier.ceylon index 7fda66f..746d003 100644 --- a/source/herd/asynctest/rule/Verifier.ceylon +++ b/source/herd/asynctest/rule/Verifier.ceylon @@ -12,7 +12,7 @@ import herd.asynctest.match { see( `function AsyncTestContext.assertThat`, `package herd.asynctest.match`, `class VerifyRule` ) tagged( "Statement" ) since ( "0.6.0" ) by( "Lis" ) shared class Verifier ( - "Element source, actually called when statement is evaluated" Element() source, + "Element source, actually called when the statement is evaluated" Element() source, "Matcher to verify stored value." Matcher matcher, "Optional title to be shown within test name." String title = "", "`True` if success to be reported otherwise only failure is reported." Boolean reportSuccess = false From 14e2044fee22808c35ee9221d4d8d2a55171955f Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 7 Dec 2016 00:30:45 +0300 Subject: [PATCH 45/72] test rule in order to restrict access to current test only --- .../herd/asynctest/rule/GaugeStatement.ceylon | 44 ++++++++++++------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/source/herd/asynctest/rule/GaugeStatement.ceylon b/source/herd/asynctest/rule/GaugeStatement.ceylon index ebc7d6e..099b8ab 100644 --- a/source/herd/asynctest/rule/GaugeStatement.ceylon +++ b/source/herd/asynctest/rule/GaugeStatement.ceylon @@ -1,5 +1,6 @@ import herd.asynctest { - AsyncTestContext + AsyncTestContext, + AsyncPrePostContext } import herd.asynctest.match { Matcher, @@ -17,43 +18,56 @@ import ceylon.collection { i.e. when [[gauge]] method is called. If matcher rejects verification the fail report is added to the final test report." see( `function AsyncTestContext.assertThat`, `package herd.asynctest.match` ) -tagged( "Statement" ) since ( "0.6.0" ) by( "Lis" ) -shared class GaugeStatement( Matcher matcher ) satisfies TestStatement +tagged( "Statement", "TestRule" ) since ( "0.6.0" ) by( "Lis" ) +shared class GaugeStatement( Matcher matcher ) satisfies TestStatement & TestRule { - "Provides synchronized access to `errors`." - ReentrantLock locker = ReentrantLock(); - "Errors from matcher during the current execution." - ArrayList errors = ArrayList(); + + "Only current test has access." + class Box() { + "Provides synchronized access to `errors`." + shared ReentrantLock locker = ReentrantLock(); + "Errors from matcher during the current execution." + shared ArrayList errors = ArrayList(); + } + + CurrentTestStore stored = CurrentTestStore( Box ); "Verifies `element` against the given `matcher`. Thread-safe." shared void gauge( Element|Element() element ) { - locker.lock(); + Box box = stored.element; + box.locker.lock(); try { MatchResult res = matcher.match( if ( is Element() element ) then element() else element ); if ( !res.accepted ) { - errors.add( AssertionError( res.string ) ); + box.errors.add( AssertionError( res.string ) ); } } catch ( Throwable err ) { - errors.add( err ); + box.errors.add( err ); } - finally { locker.unlock(); } + finally { box.locker.unlock(); } } shared actual void apply( AsyncTestContext context ) { - locker.lock(); + Box box = stored.element; + box.locker.lock(); try { - for ( item in errors ) { + for ( item in box.errors ) { context.fail( item ); } - errors.clear(); } finally { - locker.unlock(); + box.errors.clear(); + box.locker.unlock(); context.complete(); } } + + shared actual void after( AsyncPrePostContext context ) => stored.after( context ); + + shared actual void before( AsyncPrePostContext context ) => stored.before( context ); + } From 73c306a812ce6722470b452934f16a3dc615c590 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 7 Dec 2016 22:30:58 +0300 Subject: [PATCH 46/72] rename CurrentThread.isAlive method --- source/herd/asynctest/internal/CurrentThread.java | 2 +- source/herd/asynctest/rule/ChainedPrePostContext.ceylon | 2 +- source/herd/asynctest/rule/ChainedTestContext.ceylon | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/herd/asynctest/internal/CurrentThread.java b/source/herd/asynctest/internal/CurrentThread.java index 23a9551..2b96399 100644 --- a/source/herd/asynctest/internal/CurrentThread.java +++ b/source/herd/asynctest/internal/CurrentThread.java @@ -3,7 +3,7 @@ import java.lang.Thread; public class CurrentThread { - public static boolean isWorks() { + public static boolean isAlive() { Thread thr = Thread.currentThread(); return !thr.isInterrupted() && thr.isAlive(); } diff --git a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon index 54f76fd..09ab4a5 100644 --- a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon +++ b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon @@ -57,7 +57,7 @@ class ChainedPrePostContext( AsyncPrePostContext context, Iterator Date: Thu, 8 Dec 2016 20:32:41 +0300 Subject: [PATCH 47/72] provide number of events to `tick` --- source/herd/asynctest/rule/MeterRule.ceylon | 22 +++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/source/herd/asynctest/rule/MeterRule.ceylon b/source/herd/asynctest/rule/MeterRule.ceylon index d401c8d..1be21e9 100644 --- a/source/herd/asynctest/rule/MeterRule.ceylon +++ b/source/herd/asynctest/rule/MeterRule.ceylon @@ -58,20 +58,30 @@ tagged( "TestRule" ) shared class MeterRule() satisfies TestRule see( `function tick` ) shared void start() => store.element.previousTime.set( system.nanoseconds ); - "Adds clock sample from previous `tick` or from `start`. + "Adds [[numberOfTicks]] meter samples from previous `tick` or from `start`. [[start]] has to be called before the first call of `tick`. - in order to start again call `start` again." + in order to start again call `start` again. + Meter samples are: + * Time spent from previous `tick` call divided by [[numberOfTicks]], i.e. mean time required by a one event. + Added to [[timeStatistic]]. + * Rate i.e. number of events per time occured from previous `tick` call. + Number of events is equal to [[numberOfTicks]]. Added to [[rateStatistic]]. + " throws ( `class AssertionError`, "If called before `start`." ) - see( `function start` ) - shared void tick() { + see( `function start`, `value timeStatistic`, `value rateStatistic` ) + shared void tick( "Number of ticks. To be > 0." Integer numberOfTicks = 1 ) { "MeterRule: calling `tick` before `start`." assert ( store.element.previousTime.get() >= 0 ); + "MeterRule: number of ticks has to be > 0." + assert ( numberOfTicks > 0 ); Box box = store.element; Integer now = system.nanoseconds; Integer delta = system.nanoseconds - box.previousTime.getAndSet( now ); - box.timeCalculator.sample( delta / 1000000.0 ); - box.rateCalculator.sample( 1000000000.0 / delta ); + if ( delta > 0 ) { + box.timeCalculator.sample( delta / 1000000.0 / numberOfTicks ); + box.rateCalculator.sample( 1000000000.0 / delta * numberOfTicks ); + } } From fba986747ea4bb27c8cd7a4a5a79ebec949d86c7 Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 13 Dec 2016 20:32:23 +0300 Subject: [PATCH 48/72] deprecate message context, introduce runner context --- source/herd/asynctest/AsyncTestContext.ceylon | 24 ++++++++++++++++++- source/herd/asynctest/GuardTester.ceylon | 5 +++- source/herd/asynctest/Tester.ceylon | 7 +++--- .../AsyncRunnerContext.ceylon} | 8 +++---- .../asynctest/runner/AsyncTestRunner.ceylon | 5 ++-- .../asynctest/runner/ChainedRunner.ceylon | 7 +++--- .../asynctest/runner/CollectorContext.ceylon | 5 ++-- .../runner/ErrorCollectorRunner.ceylon | 3 +-- .../herd/asynctest/runner/RepeatRunner.ceylon | 3 +-- 9 files changed, 44 insertions(+), 23 deletions(-) rename source/herd/asynctest/{AsyncMessageContext.ceylon => runner/AsyncRunnerContext.ceylon} (79%) diff --git a/source/herd/asynctest/AsyncTestContext.ceylon b/source/herd/asynctest/AsyncTestContext.ceylon index d60a61a..eae8eaa 100644 --- a/source/herd/asynctest/AsyncTestContext.ceylon +++ b/source/herd/asynctest/AsyncTestContext.ceylon @@ -34,9 +34,31 @@ import herd.asynctest.match { " see( `class AsyncTestExecutor`, `package herd.asynctest.match` ) since( "0.0.1" ) by( "Lis" ) -shared interface AsyncTestContext satisfies AsyncMessageContext +shared interface AsyncTestContext// satisfies AsyncMessageContext { + "Completes the testing. To be called by the test function when testing is completed. + This wakes up test thread and allows to continue testing and store results." + shared formal void complete ( + "Optional title which is added to test variant name only if test is succeeded." + String title = "" + ); + + + "Succeeds the test with the given `message`." + shared formal void succeed( "Success message." String message ); + + + "Fails the test with either `AssertionError` or `Exception`." + shared formal void fail ( + "Reason fails this test or a function which throws either `AssertionError` or `Exception`. + If the function doesn't throw no any message is reported." + Throwable | Anything() exceptionSource, + "Optional title to be shown within test name." + String title = "" + ); + + "Verifies matcher." since( "0.6.0" ) void verifyMatcher ( "Value source: value itself or function returned value to be matched." diff --git a/source/herd/asynctest/GuardTester.ceylon b/source/herd/asynctest/GuardTester.ceylon index d6a890c..3aab615 100644 --- a/source/herd/asynctest/GuardTester.ceylon +++ b/source/herd/asynctest/GuardTester.ceylon @@ -4,6 +4,9 @@ import herd.asynctest.internal { import java.lang { Thread } +import herd.asynctest.runner { + AsyncRunnerContext +} "The most lower level of the runners - invokes test function itself. @@ -19,7 +22,7 @@ class GuardTester ( "Function to be executed." TestFunction testFunction, "Context the test function to be executed on." - AsyncMessageContext context + AsyncRunnerContext context ) extends ContextBase() satisfies AsyncTestContext { diff --git a/source/herd/asynctest/Tester.ceylon b/source/herd/asynctest/Tester.ceylon index bc90123..b212704 100644 --- a/source/herd/asynctest/Tester.ceylon +++ b/source/herd/asynctest/Tester.ceylon @@ -18,7 +18,8 @@ import herd.asynctest.internal { } import herd.asynctest.runner { - AsyncTestRunner + AsyncTestRunner, + AsyncRunnerContext } import java.util.concurrent.locks { @@ -37,7 +38,7 @@ class Tester ( "Group to run test function, is used in order to interrupt for timeout and treat uncaught exceptions." ContextThreadGroup group ) - satisfies AsyncMessageContext + satisfies AsyncRunnerContext { "Synchronizes output writing." @@ -116,7 +117,7 @@ class Tester ( "Runs all test functions using guard context." see( `class GuardTester` ) - void runWithGuard( AsyncMessageContext context ) { + void runWithGuard( AsyncRunnerContext context ) { // new GuardTest has to be used for each run for ( statement in testFunctions ) { GuardTester guard = GuardTester( group, statement, context ); diff --git a/source/herd/asynctest/AsyncMessageContext.ceylon b/source/herd/asynctest/runner/AsyncRunnerContext.ceylon similarity index 79% rename from source/herd/asynctest/AsyncMessageContext.ceylon rename to source/herd/asynctest/runner/AsyncRunnerContext.ceylon index 89c1a69..e960c24 100644 --- a/source/herd/asynctest/AsyncMessageContext.ceylon +++ b/source/herd/asynctest/runner/AsyncRunnerContext.ceylon @@ -2,11 +2,11 @@ "Base interface to push test messages to. The interface is mainly used by test runners, see package [[package herd.asynctest.runner]] for details. - Test function receives derived interface [[AsyncTestContext]]." -since( "0.6.0" ) by( "Lis" ) -shared interface AsyncMessageContext { + Test function receives [[herd.asynctest::AsyncTestContext]]." +since( "0.6.1" ) by( "Lis" ) +shared interface AsyncRunnerContext { - "Completes the testing. To be called by test function when testing is completed. + "Completes the testing. To be called by the test function when testing is completed. This wakes up test thread and allows to continue testing and store results." shared formal void complete ( "Optional title which is added to test variant name only if test is succeeded." diff --git a/source/herd/asynctest/runner/AsyncTestRunner.ceylon b/source/herd/asynctest/runner/AsyncTestRunner.ceylon index e050e60..8d81361 100644 --- a/source/herd/asynctest/runner/AsyncTestRunner.ceylon +++ b/source/herd/asynctest/runner/AsyncTestRunner.ceylon @@ -1,5 +1,4 @@ import herd.asynctest { - AsyncMessageContext, TestInfo } @@ -25,9 +24,9 @@ shared interface AsyncTestRunner { " shared formal void run ( "Context to run the test function with." - AsyncMessageContext context, + AsyncRunnerContext context, "Test function to be run." - void testing( AsyncMessageContext context ), + void testing( AsyncRunnerContext context ), "Information on the currently run test variant." TestInfo info ); diff --git a/source/herd/asynctest/runner/ChainedRunner.ceylon b/source/herd/asynctest/runner/ChainedRunner.ceylon index 6cc1556..7c90391 100644 --- a/source/herd/asynctest/runner/ChainedRunner.ceylon +++ b/source/herd/asynctest/runner/ChainedRunner.ceylon @@ -1,5 +1,4 @@ import herd.asynctest { - AsyncMessageContext, TestInfo } @@ -18,12 +17,12 @@ shared class ChainedRunner ( satisfies AsyncTestRunner { - void runnerCaller( AsyncTestRunner runner, Anything(AsyncMessageContext) testing, TestInfo info )( AsyncMessageContext context ) { + void runnerCaller( AsyncTestRunner runner, Anything(AsyncRunnerContext) testing, TestInfo info )( AsyncRunnerContext context ) { runner.run( context, testing, info ); } - shared actual void run( AsyncMessageContext context, void testing(AsyncMessageContext context), TestInfo info ) { - variable Anything(AsyncMessageContext) runTest = testing; + shared actual void run( AsyncRunnerContext context, void testing(AsyncRunnerContext context), TestInfo info ) { + variable Anything(AsyncRunnerContext) runTest = testing; for ( item in runners.reversed ) { runTest = runnerCaller( item, runTest, info ); } diff --git a/source/herd/asynctest/runner/CollectorContext.ceylon b/source/herd/asynctest/runner/CollectorContext.ceylon index b66d787..46bc1c7 100644 --- a/source/herd/asynctest/runner/CollectorContext.ceylon +++ b/source/herd/asynctest/runner/CollectorContext.ceylon @@ -1,5 +1,4 @@ import herd.asynctest { - AsyncMessageContext, VariantResultBuilder, TestVariantResult } @@ -9,7 +8,7 @@ import herd.asynctest { To get results of the test run examine [[variantResult]]." tagged( "Context" ) since( "0.6.0" ) by( "Lis" ) -shared class CollectorContext() satisfies AsyncMessageContext { +shared class CollectorContext() satisfies AsyncRunnerContext { VariantResultBuilder builder = VariantResultBuilder(); @@ -41,7 +40,7 @@ shared class CollectorContext() satisfies AsyncMessageContext { To get results of the test run examine [[variantResult]]." tagged( "Context" ) since( "0.6.0" ) by( "Lis" ) -shared class CollectAndDelegateContext( "Delegate report to" AsyncMessageContext delegateTo ) +shared class CollectAndDelegateContext( "Delegate report to" AsyncRunnerContext delegateTo ) extends CollectorContext() { diff --git a/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon b/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon index 18c5f65..8ef438a 100644 --- a/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon +++ b/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon @@ -1,5 +1,4 @@ import herd.asynctest { - AsyncMessageContext, TestInfo } import ceylon.test.engine { @@ -19,7 +18,7 @@ tagged( "Runner" ) since( "0.6.0" ) by( "Lis" ) shared class ErrorCollectorRunner() satisfies AsyncTestRunner { - shared actual void run( AsyncMessageContext context, void testing(AsyncMessageContext context), TestInfo info ) { + shared actual void run( AsyncRunnerContext context, void testing(AsyncRunnerContext context), TestInfo info ) { CollectorContext collect = CollectorContext(); collect.start(); testing( collect ); diff --git a/source/herd/asynctest/runner/RepeatRunner.ceylon b/source/herd/asynctest/runner/RepeatRunner.ceylon index 408cd4d..ea1481c 100644 --- a/source/herd/asynctest/runner/RepeatRunner.ceylon +++ b/source/herd/asynctest/runner/RepeatRunner.ceylon @@ -1,5 +1,4 @@ import herd.asynctest { - AsyncMessageContext, TestInfo, retry, TestVariantResult @@ -37,7 +36,7 @@ shared abstract class RepeatRunner() shared formal class Repeater() satisfies RepeatStrategy {} - shared actual void run( AsyncMessageContext context, void testing(AsyncMessageContext context), TestInfo info ) { + shared actual void run( AsyncRunnerContext context, void testing(AsyncRunnerContext context), TestInfo info ) { CollectorContext collect = CollectorContext(); RepeatStrategy strategy = Repeater(); while ( true ) { From 58887966d789baf5028daba326cec9b2f74c800b Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 18 Dec 2016 17:10:55 +0300 Subject: [PATCH 49/72] combinatorial --- README.md | 2 +- .../asynctest/fibonacci/fibonacciTest.ceylon | 7 +- .../CeylonJavaMapMicrobenchmark.ceylon | 9 +- .../asynctest/matchers/AllOfTest.ceylon | 4 +- .../asynctest/matchers/AndCompare.ceylon | 4 +- .../asynctest/matchers/Compare.ceylon | 6 +- .../asynctest/matchers/OneOfTest.ceylon | 4 +- .../asynctest/matchers/OrCompare.ceylon | 4 +- .../asynctest/matchers/RangeCompare.ceylon | 4 +- .../asynctest/matchers/SomeOfTest.ceylon | 4 +- .../asynctest/matchers/ValueEquality.ceylon | 6 +- .../asynctest/matchers/listMatchers.ceylon | 6 +- .../asynctest/matchers/testParams.ceylon | 2 +- .../herd/examples/asynctest/module.ceylon | 12 +- .../parameterized/combinatorial.ceylon | 46 +++++++ .../package.ceylon | 6 +- .../typeParameterized.ceylon | 8 +- .../scheduler/SchedulerTester.ceylon | 10 +- .../herd/asynctest/AsyncTestExecutor.ceylon | 7 +- .../herd/asynctest/AsyncTestProcessor.ceylon | 30 ++++- .../herd/asynctest/TestGroupExecutor.ceylon | 20 +-- source/herd/asynctest/TestInfo.ceylon | 10 +- source/herd/asynctest/TestOutput.ceylon | 5 +- source/herd/asynctest/annotations.ceylon | 106 +-------------- source/herd/asynctest/asyncTestRunner.ceylon | 77 +---------- .../internal/declarationVerifier.ceylon | 78 +++++++++++ .../{ => internal}/extractSourceValue.ceylon | 10 +- .../internal/resolveArgumentList.ceylon | 27 ++++ source/herd/asynctest/module.ceylon | 52 ++------ .../parameterization/ArgumentVariants.ceylon | 19 +++ .../parameterization/CombinatorialKind.ceylon | 31 +++++ .../parameterization/MixingEnumerator.ceylon | 124 ++++++++++++++++++ .../{ => parameterization}/TestVariant.ceylon | 21 +-- .../TestVariantEnumerator.ceylon | 61 ++++----- .../TestVariantProvider.ceylon | 3 +- .../parameterization/combinatorial.ceylon | 94 +++++++++++++ .../parameterization/dataSource.ceylon | 61 +++++++++ .../mixingCombinations.ceylon | 72 ++++++++++ .../asynctest/parameterization/package.ceylon | 120 +++++++++++++++++ .../parameterization/parameterized.ceylon | 110 ++++++++++++++++ .../permutingCombinations.ceylon | 102 ++++++++++++++ .../zippingCombinations.ceylon | 76 +++++++++++ source/herd/asynctest/resolveArguments.ceylon | 43 ------ .../herd/asynctest/variantEnumerators.ceylon | 70 ++++++++++ 44 files changed, 1188 insertions(+), 385 deletions(-) create mode 100644 examples/herd/examples/asynctest/parameterized/combinatorial.ceylon rename examples/herd/examples/asynctest/{generics => parameterized}/package.ceylon (65%) rename examples/herd/examples/asynctest/{generics => parameterized}/typeParameterized.ceylon (97%) create mode 100644 source/herd/asynctest/internal/declarationVerifier.ceylon rename source/herd/asynctest/{ => internal}/extractSourceValue.ceylon (55%) create mode 100644 source/herd/asynctest/internal/resolveArgumentList.ceylon create mode 100644 source/herd/asynctest/parameterization/ArgumentVariants.ceylon create mode 100644 source/herd/asynctest/parameterization/CombinatorialKind.ceylon create mode 100644 source/herd/asynctest/parameterization/MixingEnumerator.ceylon rename source/herd/asynctest/{ => parameterization}/TestVariant.ceylon (84%) rename source/herd/asynctest/{ => parameterization}/TestVariantEnumerator.ceylon (53%) rename source/herd/asynctest/{ => parameterization}/TestVariantProvider.ceylon (91%) create mode 100644 source/herd/asynctest/parameterization/combinatorial.ceylon create mode 100644 source/herd/asynctest/parameterization/dataSource.ceylon create mode 100644 source/herd/asynctest/parameterization/mixingCombinations.ceylon create mode 100644 source/herd/asynctest/parameterization/package.ceylon create mode 100644 source/herd/asynctest/parameterization/parameterized.ceylon create mode 100644 source/herd/asynctest/parameterization/permutingCombinations.ceylon create mode 100644 source/herd/asynctest/parameterization/zippingCombinations.ceylon delete mode 100644 source/herd/asynctest/resolveArguments.ceylon create mode 100644 source/herd/asynctest/variantEnumerators.ceylon diff --git a/README.md b/README.md index 6cdd2eb..370b591 100644 --- a/README.md +++ b/README.md @@ -45,5 +45,5 @@ See usage details in [API documentation](https://modules.ceylon-lang.org/repo/1/ * [Microbenchmark](examples/herd/examples/asynctest/mapperformance) - comparative performance test of Ceylon / Java HashMap and TreeMap. * [Matchers](examples/herd/examples/asynctest/matchers) - matchers usage. -* [Generics](examples/herd/examples/asynctest/generics) - type-parameterized testing. +* [Parameterized](examples/herd/examples/asynctest/parameterized) - type- and value- parameterized testing. * [Rules](examples/herd/examples/asynctest/rule) - test rules usage. diff --git a/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon b/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon index 941e616..42d6db5 100644 --- a/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon +++ b/examples/herd/examples/asynctest/fibonacci/fibonacciTest.ceylon @@ -3,11 +3,8 @@ import ceylon.test { test } import herd.asynctest { - AsyncTestContext, - parameterized, async, - TestVariant, timeout } import herd.asynctest.match { @@ -15,6 +12,10 @@ import herd.asynctest.match { EqualTo, Mapping } +import herd.asynctest.parameterization { + TestVariant, + parameterized +} "Fibonacci test parameters." diff --git a/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon b/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon index c3a72a9..e6af210 100644 --- a/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon +++ b/examples/herd/examples/asynctest/mapperformance/CeylonJavaMapMicrobenchmark.ceylon @@ -1,9 +1,6 @@ import herd.asynctest { - AsyncTestContext, - parameterized, - arguments, - TestVariant + arguments } import ceylon.test { @@ -41,6 +38,10 @@ import herd.asynctest.rule { MeterRule, AtomicValueRule } +import herd.asynctest.parameterization { + parameterized, + TestVariant +} "Test parameters: diff --git a/examples/herd/examples/asynctest/matchers/AllOfTest.ceylon b/examples/herd/examples/asynctest/matchers/AllOfTest.ceylon index 6a0ca10..9483b1d 100644 --- a/examples/herd/examples/asynctest/matchers/AllOfTest.ceylon +++ b/examples/herd/examples/asynctest/matchers/AllOfTest.ceylon @@ -12,7 +12,9 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, + AsyncTestContext +} +import herd.asynctest.parameterization { parameterized } diff --git a/examples/herd/examples/asynctest/matchers/AndCompare.ceylon b/examples/herd/examples/asynctest/matchers/AndCompare.ceylon index 207d53a..307d9a6 100644 --- a/examples/herd/examples/asynctest/matchers/AndCompare.ceylon +++ b/examples/herd/examples/asynctest/matchers/AndCompare.ceylon @@ -11,7 +11,9 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, + AsyncTestContext +} +import herd.asynctest.parameterization { parameterized } diff --git a/examples/herd/examples/asynctest/matchers/Compare.ceylon b/examples/herd/examples/asynctest/matchers/Compare.ceylon index c1aecb4..9163a9e 100644 --- a/examples/herd/examples/asynctest/matchers/Compare.ceylon +++ b/examples/herd/examples/asynctest/matchers/Compare.ceylon @@ -4,8 +4,7 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, - parameterized + AsyncTestContext } import herd.asynctest.match { @@ -16,6 +15,9 @@ import herd.asynctest.match { Less, Greater } +import herd.asynctest.parameterization { + parameterized +} class Compare() { diff --git a/examples/herd/examples/asynctest/matchers/OneOfTest.ceylon b/examples/herd/examples/asynctest/matchers/OneOfTest.ceylon index bd53c2a..eccc8f1 100644 --- a/examples/herd/examples/asynctest/matchers/OneOfTest.ceylon +++ b/examples/herd/examples/asynctest/matchers/OneOfTest.ceylon @@ -12,7 +12,9 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, + AsyncTestContext +} +import herd.asynctest.parameterization { parameterized } diff --git a/examples/herd/examples/asynctest/matchers/OrCompare.ceylon b/examples/herd/examples/asynctest/matchers/OrCompare.ceylon index 0d7acb6..f5d2158 100644 --- a/examples/herd/examples/asynctest/matchers/OrCompare.ceylon +++ b/examples/herd/examples/asynctest/matchers/OrCompare.ceylon @@ -11,7 +11,9 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, + AsyncTestContext +} +import herd.asynctest.parameterization { parameterized } diff --git a/examples/herd/examples/asynctest/matchers/RangeCompare.ceylon b/examples/herd/examples/asynctest/matchers/RangeCompare.ceylon index ebfa2aa..9451b0c 100644 --- a/examples/herd/examples/asynctest/matchers/RangeCompare.ceylon +++ b/examples/herd/examples/asynctest/matchers/RangeCompare.ceylon @@ -13,7 +13,9 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, + AsyncTestContext +} +import herd.asynctest.parameterization { parameterized } diff --git a/examples/herd/examples/asynctest/matchers/SomeOfTest.ceylon b/examples/herd/examples/asynctest/matchers/SomeOfTest.ceylon index 9531940..67fa2a7 100644 --- a/examples/herd/examples/asynctest/matchers/SomeOfTest.ceylon +++ b/examples/herd/examples/asynctest/matchers/SomeOfTest.ceylon @@ -12,7 +12,9 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, + AsyncTestContext +} +import herd.asynctest.parameterization { parameterized } diff --git a/examples/herd/examples/asynctest/matchers/ValueEquality.ceylon b/examples/herd/examples/asynctest/matchers/ValueEquality.ceylon index 3b6d421..1e96a25 100644 --- a/examples/herd/examples/asynctest/matchers/ValueEquality.ceylon +++ b/examples/herd/examples/asynctest/matchers/ValueEquality.ceylon @@ -1,6 +1,5 @@ import herd.asynctest { - AsyncTestContext, - parameterized + AsyncTestContext } import ceylon.test { test @@ -8,6 +7,9 @@ import ceylon.test { import herd.asynctest.match { ValueEquality } +import herd.asynctest.parameterization { + parameterized +} "Checks value equality for `Integer` and `String`." diff --git a/examples/herd/examples/asynctest/matchers/listMatchers.ceylon b/examples/herd/examples/asynctest/matchers/listMatchers.ceylon index 859f672..8075bee 100644 --- a/examples/herd/examples/asynctest/matchers/listMatchers.ceylon +++ b/examples/herd/examples/asynctest/matchers/listMatchers.ceylon @@ -4,8 +4,7 @@ import ceylon.test { } import herd.asynctest { - AsyncTestContext, - parameterized + AsyncTestContext } import herd.asynctest.match { @@ -14,6 +13,9 @@ import herd.asynctest.match { Beginning, Finishing } +import herd.asynctest.parameterization { + parameterized +} "Verify List Matchers" diff --git a/examples/herd/examples/asynctest/matchers/testParams.ceylon b/examples/herd/examples/asynctest/matchers/testParams.ceylon index 3d19a2c..360502e 100644 --- a/examples/herd/examples/asynctest/matchers/testParams.ceylon +++ b/examples/herd/examples/asynctest/matchers/testParams.ceylon @@ -1,5 +1,5 @@ -import herd.asynctest { +import herd.asynctest.parameterization { TestVariant } diff --git a/examples/herd/examples/asynctest/module.ceylon b/examples/herd/examples/asynctest/module.ceylon index 008f2d2..3be0d6a 100644 --- a/examples/herd/examples/asynctest/module.ceylon +++ b/examples/herd/examples/asynctest/module.ceylon @@ -1,11 +1,11 @@ "Contains some examples of `asyncTest` usage: - * [[package herd.examples.asynctest.fibonacci]] - test of calculation of Fibonacci numbers in separated thread - * [[package herd.examples.asynctest.generics]] - type-parameterized testing - * [[package herd.examples.asynctest.mapperformance]] - comparative performance test of Ceylon - Java HashMap and TreeMap - * [[package herd.examples.asynctest.matchers]] - several examples of matchers from `package herd.asynctest.match` - * [[package herd.examples.asynctest.rule]] - usage of the test rules. - * [[package herd.examples.asynctest.scheduler]] - time scheduler testing + * [[package herd.examples.asynctest.fibonacci]] - test of calculation of Fibonacci numbers in separated thread + * [[package herd.examples.asynctest.mapperformance]] - comparative performance test of Ceylon - Java HashMap and TreeMap + * [[package herd.examples.asynctest.matchers]] - several examples of matchers from `package herd.asynctest.match` + * [[package herd.examples.asynctest.parameterized]] - type- and value- parameterized testing + * [[package herd.examples.asynctest.rule]] - usage of the test rules + * [[package herd.examples.asynctest.scheduler]] - time scheduler testing " license ( " diff --git a/examples/herd/examples/asynctest/parameterized/combinatorial.ceylon b/examples/herd/examples/asynctest/parameterized/combinatorial.ceylon new file mode 100644 index 0000000..97afefa --- /dev/null +++ b/examples/herd/examples/asynctest/parameterized/combinatorial.ceylon @@ -0,0 +1,46 @@ +import herd.asynctest.parameterization { + zipping, + zippedSource, + permutationSource, + mixing +} +import herd.asynctest { + AsyncTestContext +} +import herd.asynctest.match { + Greater +} +import ceylon.test { + test +} + + +Integer[] firstArgument => [1,2,3]; +Integer[] secondArgument => [10,20,30]; +String[] signArgument => ["+","-"]; + + +"Zipping testing." +shared test zipping +void testZipping ( + AsyncTestContext context, + zippedSource(`value firstArgument`) Integer arg1, + zippedSource(`value secondArgument`) Integer arg2 +) { + context.assertThat(arg2, Greater(arg1), "", true); + context.complete(); +} + + +"Mixing testing." +shared test mixing +void testMixing ( + AsyncTestContext context, + zippedSource(`value firstArgument`) Integer arg1, + permutationSource(`value signArgument`) String arg2, + zippedSource(`value secondArgument`) Integer arg3 +) { + context.succeed( "``arg1````arg2````arg3``" ); + context.complete(); +} + diff --git a/examples/herd/examples/asynctest/generics/package.ceylon b/examples/herd/examples/asynctest/parameterized/package.ceylon similarity index 65% rename from examples/herd/examples/asynctest/generics/package.ceylon rename to examples/herd/examples/asynctest/parameterized/package.ceylon index 213a239..ccf0ee6 100644 --- a/examples/herd/examples/asynctest/generics/package.ceylon +++ b/examples/herd/examples/asynctest/parameterized/package.ceylon @@ -1,14 +1,14 @@ import herd.asynctest { - + async, concurrent } " - Type parameterized test. + Type- and value- parameterized test. Annotated with `async` - all test functions of the package to be run using `AsyncTestExecutor`. " by( "Lis" ) async concurrent -shared package herd.examples.asynctest.generics; +shared package herd.examples.asynctest.parameterized; diff --git a/examples/herd/examples/asynctest/generics/typeParameterized.ceylon b/examples/herd/examples/asynctest/parameterized/typeParameterized.ceylon similarity index 97% rename from examples/herd/examples/asynctest/generics/typeParameterized.ceylon rename to examples/herd/examples/asynctest/parameterized/typeParameterized.ceylon index 4d723a8..ba2c811 100644 --- a/examples/herd/examples/asynctest/generics/typeParameterized.ceylon +++ b/examples/herd/examples/asynctest/parameterized/typeParameterized.ceylon @@ -2,13 +2,15 @@ import ceylon.test { test } import herd.asynctest { - parameterized, - AsyncTestContext, - TestVariant + AsyncTestContext } import herd.asynctest.match { EqualTo } +import herd.asynctest.parameterization { + parameterized, + TestVariant +} "Parameters of [[testIdentity]]." diff --git a/examples/herd/examples/asynctest/scheduler/SchedulerTester.ceylon b/examples/herd/examples/asynctest/scheduler/SchedulerTester.ceylon index 7c271f6..54f4a2c 100644 --- a/examples/herd/examples/asynctest/scheduler/SchedulerTester.ceylon +++ b/examples/herd/examples/asynctest/scheduler/SchedulerTester.ceylon @@ -1,11 +1,8 @@ import herd.asynctest { - AsyncTestContext, - parameterized, async, arguments, - AsyncPrePostContext, - TestVariant + AsyncPrePostContext } import ceylon.test { @@ -22,7 +19,10 @@ import java.lang { Runtime } - +import herd.asynctest.parameterization { + parameterized, + TestVariant +} shared {TestVariant*} testTimers => { diff --git a/source/herd/asynctest/AsyncTestExecutor.ceylon b/source/herd/asynctest/AsyncTestExecutor.ceylon index 7724e28..d0f5b15 100644 --- a/source/herd/asynctest/AsyncTestExecutor.ceylon +++ b/source/herd/asynctest/AsyncTestExecutor.ceylon @@ -12,6 +12,7 @@ import ceylon.test.engine.spi { } + "Async test executor. #### Capabilities @@ -20,14 +21,12 @@ import ceylon.test.engine.spi { * running test functions concurrently or sequentialy, see [[concurrent]] annotation and [[module herd.asynctest]] * multi-reporting: several failures or successes can be reported for a one test execution, each report is represented as test variant and might be marked with `String` title - * value- and type- parameterized testing with a set of function arguments, see [[parameterized]] for details + * value- and type- parameterized testing with a set of function arguments, see [[package herd.asynctest.parameterization]] for details * conditional execution with annotations satisfied `ceylon.test.engine.spi::TestCondition` interface In order to utilize this executor capabilities test function has to accept [[AsyncTestContext]] as the first argument: test async - void doTesting(AsyncTestContext context) {...} - - > Test function may have more arguments if it is annotated with [[parameterized]] annotation. + void doTesting(AsyncTestContext context) {...} #### Running diff --git a/source/herd/asynctest/AsyncTestProcessor.ceylon b/source/herd/asynctest/AsyncTestProcessor.ceylon index 6414344..1bd5de0 100644 --- a/source/herd/asynctest/AsyncTestProcessor.ceylon +++ b/source/herd/asynctest/AsyncTestProcessor.ceylon @@ -29,9 +29,10 @@ import ceylon.language.meta { type } import herd.asynctest.internal { - ContextThreadGroup, - findFirstAnnotation + findFirstAnnotation, + extractSourceValue, + declarationVerifier } import herd.asynctest.runner { @@ -40,6 +41,11 @@ import herd.asynctest.runner { RepeatStrategy, repeatOnce } +import herd.asynctest.parameterization { + TestVariantEnumerator, + TestVariant, + TestVariantProvider +} "Processes test execution with the branch test generic parameters and function arguments." @@ -61,7 +67,7 @@ class AsyncTestProcessor( Integer timeOutMilliseconds = extractTimeout( testFunctionDeclaration ); "`true` if test function is run on async test context." - Boolean runOnAsyncContext = asyncTestRunner.isAsyncDeclaration( testFunctionDeclaration ); + Boolean runOnAsyncContext = declarationVerifier.isAsyncDeclaration( testFunctionDeclaration ); "Init context to perform test initialization." PrePostContext prePostContext = PrePostContext( group ); @@ -70,6 +76,22 @@ class AsyncTestProcessor( Tester tester = Tester( group ); + "Resolves a list of type parameters and function arguments provided by annotations satisfied `TestVariantProvider`. + Returns a list of parameters list." + TestVariantEnumerator resolveParameterizedList() { + value providers = testFunctionDeclaration.annotations().narrow(); + if ( providers.empty ) { + return EmptyTestVariantEnumerator(); + } + else if ( providers.size == 1 ) { + return providers.first?.variants( testFunctionDeclaration, instance ) else EmptyTestVariantEnumerator(); + } + else { + return CombinedVariantEnumerator( providers*.variants( testFunctionDeclaration, instance ).iterator() ); + } + } + + "Extracts test runner from test function annotations." AsyncTestRunner? getRunner() => if ( exists ann = findFirstAnnotation( testFunctionDeclaration ) ) @@ -190,7 +212,7 @@ class AsyncTestProcessor( } else { // execute test with the given number of test variants - return executeVariants( functionContext, resolveParameterizedList( testFunctionDeclaration, instance ) ); + return executeVariants( functionContext, resolveParameterizedList() ); } } catch ( TestSkippedException e ) { diff --git a/source/herd/asynctest/TestGroupExecutor.ceylon b/source/herd/asynctest/TestGroupExecutor.ceylon index 294600a..45f8abb 100644 --- a/source/herd/asynctest/TestGroupExecutor.ceylon +++ b/source/herd/asynctest/TestGroupExecutor.ceylon @@ -51,8 +51,10 @@ import ceylon.test.event { TestStartedEvent } import herd.asynctest.internal { - - ContextThreadGroup + ContextThreadGroup, + extractSourceValue, + resolveArgumentList, + declarationVerifier } import java.util.concurrent.locks { @@ -91,7 +93,7 @@ class TestGroupExecutor ( if ( exists factory = optionalAnnotation( `FactoryAnnotation`, declaration ) ) { // factory exists - use this to instantiate object value args = resolveArgumentList( factory.factoryFunction, null ); - if ( asyncTestRunner.isAsyncFactoryDeclaration( factory.factoryFunction ) ) { + if ( declarationVerifier.isAsyncFactoryDeclaration( factory.factoryFunction ) ) { return FactoryContext( "``factory.factoryFunction.name``", group ).run ( ( AsyncFactoryContext context ) { factory.factoryFunction.apply<>().apply( context, *args ); @@ -166,7 +168,7 @@ class TestGroupExecutor ( // timeout value timeOut = extractTimeout( decl ); - if ( asyncTestRunner.isAsyncPrepostDeclaration( decl ) ) { + if ( declarationVerifier.isAsyncPrepostDeclaration( decl ) ) { return PrePostFunction ( ( AsyncPrePostContext context ) { prepostFunction.apply( context, *args ); @@ -198,7 +200,7 @@ class TestGroupExecutor ( PrePostFunction[] getSuiteInitializers( Object? instance, ClassOrInterface<>? instanceType ) { if ( exists container = instance, exists containerType = instanceType ) { // attributes marked with `testRule` - value suiteAttrs = containerType.getAttributes( asyncTestRunner.ruleAnnotationClass ); + value suiteAttrs = containerType.getAttributes( declarationVerifier.ruleAnnotationClass ); return [ for ( attr in suiteAttrs ) PrePostFunction ( attr.bind( container ).get().initialize, extractTimeoutFromObject( attr.declaration, "initialize" ), attr.declaration.name, @@ -221,7 +223,7 @@ class TestGroupExecutor ( PrePostFunction[] getTestInitializers( Object? instance, ClassOrInterface<>? instanceType ) { if ( exists container = instance, exists containerType = instanceType ) { - value attrs = containerType.getAttributes( asyncTestRunner.ruleAnnotationClass ); + value attrs = containerType.getAttributes( declarationVerifier.ruleAnnotationClass ); return [ for ( attr in attrs ) PrePostFunction ( attr.bind( container ).get().before, extractTimeoutFromObject( attr.declaration, "before" ), @@ -244,7 +246,7 @@ class TestGroupExecutor ( TestFunction[] getTestStatements( Object? instance, ClassOrInterface<>? instanceType ) { if ( exists container = instance, exists containerType = instanceType ) { - value attrs = containerType.getAttributes( asyncTestRunner.ruleAnnotationClass ); + value attrs = containerType.getAttributes( declarationVerifier.ruleAnnotationClass ); return [ for ( attr in attrs ) TestFunction ( attr.bind( container ).get().apply, extractTimeoutFromObject( attr.declaration, "apply" ), @@ -267,7 +269,7 @@ class TestGroupExecutor ( PrePostFunction[] getSuiteCleaners( Object? instance, ClassOrInterface<>? instanceType ) { if ( exists container = instance, exists containerType = instanceType ) { - value suiteRules = containerType.getAttributes( asyncTestRunner.ruleAnnotationClass ); + value suiteRules = containerType.getAttributes( declarationVerifier.ruleAnnotationClass ); return [ for ( attr in suiteRules ) PrePostFunction ( attr.bind( container ).get().dispose, extractTimeoutFromObject( attr.declaration, "dispose" ), @@ -289,7 +291,7 @@ class TestGroupExecutor ( PrePostFunction[] getTestCleaners( Object? instance, ClassOrInterface<>? instanceType ) { if ( exists container = instance, exists containerType = instanceType ) { - value attrs = containerType.getAttributes( asyncTestRunner.ruleAnnotationClass ); + value attrs = containerType.getAttributes( declarationVerifier.ruleAnnotationClass ); return [ for ( attr in attrs ) PrePostFunction ( attr.bind( container ).get().after, extractTimeoutFromObject( attr.declaration, "after" ), diff --git a/source/herd/asynctest/TestInfo.ceylon b/source/herd/asynctest/TestInfo.ceylon index 123a4eb..f8df79a 100644 --- a/source/herd/asynctest/TestInfo.ceylon +++ b/source/herd/asynctest/TestInfo.ceylon @@ -12,7 +12,7 @@ import herd.asynctest.internal { } -"Test info represents and information on currently running test variant." +"Information on currently running test variant." see( `interface AsyncPrePostContext`, `interface AsyncTestRunner` ) since( "0.6.0" ) by( "Lis" ) shared final class TestInfo ( @@ -24,8 +24,8 @@ shared final class TestInfo ( shared Anything[] arguments, "Test variant name as represented in the test report." shared String variantName, - "Time out in milliseconds for a one test function run, <= 0 if no limit." - shared Integer timeOutMilliseconds + "Timeout in milliseconds for a one test function run, if <= 0 no limit." + shared Integer timeoutMilliseconds ) { variable Integer memoizedHash = 0; @@ -36,7 +36,7 @@ shared final class TestInfo ( parameters == that.parameters && arguments == that.arguments && variantName == that.variantName && - timeOutMilliseconds == that.timeOutMilliseconds; + timeoutMilliseconds == that.timeoutMilliseconds; } else { return false; @@ -49,7 +49,7 @@ shared final class TestInfo ( memoizedHash = 31 * memoizedHash + sequenceHash( arguments, 31 ); memoizedHash = 31 * memoizedHash + testFunction.hash; memoizedHash = 31 * memoizedHash + variantName.hash; - memoizedHash = 31 * memoizedHash + timeOutMilliseconds; + memoizedHash = 31 * memoizedHash + timeoutMilliseconds; } return memoizedHash; } diff --git a/source/herd/asynctest/TestOutput.ceylon b/source/herd/asynctest/TestOutput.ceylon index febb65e..4c78070 100644 --- a/source/herd/asynctest/TestOutput.ceylon +++ b/source/herd/asynctest/TestOutput.ceylon @@ -6,6 +6,9 @@ import ceylon.test.engine.spi { TestExecutionContext } +import herd.asynctest.parameterization { + TestVariantEnumerator +} "Represents a one report." @@ -22,7 +25,7 @@ shared final class TestOutput ( "Results of a one test function run with some arguments." -see( `interface TestVariantEnumerator`, `class TestVariant`, `class VariantResultBuilder` ) +see( `interface TestVariantEnumerator`, `class VariantResultBuilder` ) since( "0.6.0" ) by( "Lis" ) shared final class TestVariantResult ( "Outputs from test." shared TestOutput[] testOutput, diff --git a/source/herd/asynctest/annotations.ceylon b/source/herd/asynctest/annotations.ceylon index da93015..9e7ad86 100644 --- a/source/herd/asynctest/annotations.ceylon +++ b/source/herd/asynctest/annotations.ceylon @@ -51,7 +51,7 @@ shared annotation TestExecutorAnnotation async() => testExecutor( `class AsyncTe see( `function arguments` ) since( "0.5.0" ) by( "Lis" ) shared final annotation class ArgumentsAnnotation ( - "The source function or value declaration which has to take no arguments and has to return a stream of values. + "The source function or value declaration which has to return a stream of values. The source may be either top-level or tested class shared member." shared FunctionOrValueDeclaration source ) @@ -62,115 +62,13 @@ shared final annotation class ArgumentsAnnotation ( "Provides arguments for a one-shot functions. See [[ArgumentsAnnotation]] for details." since( "0.5.0" ) by( "Lis" ) shared annotation ArgumentsAnnotation arguments ( - "The source function or value declaration which has to take no arguments and has to return a stream of values. + "The source function or value declaration which has to return a stream of values. The source may be either top-level or tested class shared member." FunctionOrValueDeclaration source ) => ArgumentsAnnotation( source ); - -"Indicates that generic (with possibly empty generic parameter list) test function - has to be executed with given test variants. - - The annotation provides parameterized testing based on collection of test variants. - It takes two arguments: - 1. Declaration of function or value which returns a collection of test variants `{TestVariant*}`. - 2. Number of failed variants to stop testing. Default is -1 which means no limit. - - The test will be performed using all test variants returned by the given stream - or while total number of failed variants not exceeds specified limit. - - > [[parameterized]] annotation may occur multiple times at a given test function. - > The variants source may be either top-level or tested class shared member. - - - #### Example: - - Value identity(Value argument) => argument; - - {TestVariant*} identityArgs => { - TestVariant([`String`], [\"stringIdentity\"]), - TestVariant([`Integer`], [1]), - TestVariant([`Float`], [1.0]) - }; - - shared test async - parameterized(`value identityArgs`) - void testIdentity(AsyncTestContext context, Value arg) - given Value satisfies Object - { - context.assertThat(identity(arg), EqualObjects(arg), \"\", true); - context.complete(); - } - - In the above example the function `testIdentity` will be called 3 times: - * `testIdentity(context, \"stringIdentity\");` - * `testIdentity(context, 1);` - * `testIdentity(context, 1.0);` - - In order to run test with conventional (non-generic function) type parameters list has to be empty: - [Hobbit] who => [bilbo]; - {TestVariant*} dwarves => { - TestVariant([], [fili]), - TestVariant([], [kili]), - TestVariant([], [balin], - TestVariant([], [dwalin]), - ... - }; - - arguments(`value who`) - class HobbitTester(Hobbit hobbit) { - shared test async - parameterized(`value dwarves`, 2) - void thereAndBackAgain(AsyncTestContext context, Dwarf dwarf) { - context.assertTrue(hobbit.thereAndBackAgain(dwarf)...); - context.complete(); - } - } - - In this example class `HobbitTester` is instantiated once with argument provided by value `who` and - method `thereAndBackAgain` is called multiply times according to size of `dwarves` stream. - According to second argument of `parameterized` annotation the test will be stopped - if two different invoking of `thereAndBackAgain` with two different arguments report failure. - " -since( "0.6.0" ) by( "Lis" ) -see( `function parameterized`, `class TestVariant` ) -shared final annotation class ParameterizedAnnotation ( - "The source function or value declaration which has to take no arguments and has to return a stream - of test variants: `{TestVariant*}`. - The source may be either top-level or tested class shared member." - shared FunctionOrValueDeclaration source, - "Maximum number of failed variants before stop. Unlimited if <= 0." - shared Integer maxFailedVariants -) - satisfies SequencedAnnotation & TestVariantProvider -{ - - "Returns test variant enumerator based on test variants extracted from `source`." - shared actual TestVariantEnumerator variants( FunctionDeclaration testFunction, Object? instance) - => TestVariantIterator ( - extractSourceValue<{TestVariant*}>( source, instance ).iterator().next, - maxFailedVariants - ); - -} - - -"Provides parameters for the parameterized testing. See [[ParameterizedAnnotation]] for the details." -see( `class TestVariant` ) -since( "0.6.0" ) by( "Lis" ) -shared annotation ParameterizedAnnotation parameterized ( - "The source function or value declaration which has to take no arguments and has to return - a stream of test variants: `{TestVariant*}`. - The source may be either top-level or tested class shared member." - FunctionOrValueDeclaration source, - "Maximum number of failed variants before stop. Unlimited if <= 0." - Integer maxFailedVariants = -1 -) - => ParameterizedAnnotation( source, maxFailedVariants ); - - "Indicates that class has to be instantiated using a given factory function. [[factory]] annotation takes declaration of top-level factory function. diff --git a/source/herd/asynctest/asyncTestRunner.ceylon b/source/herd/asynctest/asyncTestRunner.ceylon index 6df830d..315aa94 100644 --- a/source/herd/asynctest/asyncTestRunner.ceylon +++ b/source/herd/asynctest/asyncTestRunner.ceylon @@ -4,31 +4,14 @@ import ceylon.collection { import ceylon.language.meta.declaration { FunctionDeclaration, ClassDeclaration, - Package, - InterfaceDeclaration, - OpenInterfaceType + Package } import ceylon.test { TestDescription } -import ceylon.test.annotation { - TestExecutorAnnotation -} import ceylon.test.engine.spi { TestExecutionContext } -import herd.asynctest.rule { - - TestRuleAnnotation -} -import ceylon.language.meta.model { - - Class -} -import herd.asynctest.internal { - - findFirstAnnotation -} import java.util.concurrent { CountDownLatch, @@ -46,33 +29,16 @@ import java.util.concurrent.locks { ReentrantLock } +import herd.asynctest.internal { + findFirstAnnotation, + declarationVerifier +} "Combines tests in the suites and starts test executions." since( "0.0.1" ) by( "Lis" ) object asyncTestRunner { - "Memoization of [[TestRuleAnnotation]]." - see( `class TestGroupExecutor` ) - shared Class ruleAnnotationClass = `TestRuleAnnotation`; - - - "AsyncTestContext declaration memoization." - see( `function isAsyncDeclaration` ) - InterfaceDeclaration asyncContextDeclaration = `interface AsyncTestContext`; - - "AsyncPrePostContext declaration memoization." - see( `function isAsyncPrepostDeclaration` ) - InterfaceDeclaration prepostContextDeclaration = `interface AsyncPrePostContext`; - - "AsyncFactoryContext declaration memoization." - see( `function isAsyncFactoryDeclaration` ) - InterfaceDeclaration factoryContextDeclaration = `interface AsyncFactoryContext`; - - "Cache async test executor declaration. See `function isAsyncExecutedTest`" - ClassDeclaration asyncTestDeclaration = `class AsyncTestExecutor`; - - "Executors of test groups - package or class level." HashMap sequentialExecutors = HashMap(); HashMap concurrentExecutors = HashMap(); @@ -93,7 +59,7 @@ object asyncTestRunner { small Integer numberOfAsyncTest( TestDescription* descriptions ) { variable small Integer ret = 0; for ( descr in descriptions ) { - if ( exists fDeclaration = descr.functionDeclaration, isAsyncExecutedTest( fDeclaration ) ) { + if ( exists fDeclaration = descr.functionDeclaration, declarationVerifier.isAsyncExecutedTest( fDeclaration ) ) { ret ++; } ret += numberOfAsyncTest( *descr.children ); @@ -196,35 +162,4 @@ object asyncTestRunner { } } - - "`true` if execution is performed using `AsyncTestExecutor`" - Boolean isAsyncExecutedTest( FunctionDeclaration functionDeclaration ) => - if ( exists ann = findFirstAnnotation( functionDeclaration ) ) - then ann.executor == asyncTestDeclaration else false; - - - "Returns `true` if function takes `firstArg` as first argument." - Boolean isFirstArgSatisfies( FunctionDeclaration functionDeclaration, InterfaceDeclaration firstArg ) { - if ( is OpenInterfaceType argType = functionDeclaration.parameterDeclarations.first?.openType, - argType.declaration == firstArg - ) { - return true; - } - else { - return false; - } - } - - "Returns `true` if function runs async test == takes `AsyncTestContext` as first argument." - shared Boolean isAsyncDeclaration( FunctionDeclaration functionDeclaration ) - => isFirstArgSatisfies( functionDeclaration, asyncContextDeclaration ); - - "Returns `true` if function runs async test initialization == takes `AsyncPrePostContext` as first argument." - shared Boolean isAsyncPrepostDeclaration( FunctionDeclaration functionDeclaration ) - => isFirstArgSatisfies( functionDeclaration, prepostContextDeclaration ); - - "Returns `true` if function runs async test initialization == takes `AsyncFactoryContext` as first argument." - shared Boolean isAsyncFactoryDeclaration( FunctionDeclaration functionDeclaration ) - => isFirstArgSatisfies( functionDeclaration, factoryContextDeclaration ); - } diff --git a/source/herd/asynctest/internal/declarationVerifier.ceylon b/source/herd/asynctest/internal/declarationVerifier.ceylon new file mode 100644 index 0000000..cefa4cf --- /dev/null +++ b/source/herd/asynctest/internal/declarationVerifier.ceylon @@ -0,0 +1,78 @@ +import herd.asynctest { + AsyncPrePostContext, + AsyncTestExecutor, + AsyncFactoryContext, + AsyncTestContext +} +import herd.asynctest.rule { + TestRuleAnnotation +} +import ceylon.language.meta.declaration { + ClassDeclaration, + InterfaceDeclaration, + FunctionDeclaration, + OpenInterfaceType +} +import ceylon.language.meta.model { + Class +} +import ceylon.test.annotation { + TestExecutorAnnotation +} + + +"Verifies if declaration meets templates" +since( "0.6.1" ) by( "Lis" ) +shared object declarationVerifier { + + "Memoization of [[TestRuleAnnotation]]." + shared Class ruleAnnotationClass = `TestRuleAnnotation`; + + "AsyncTestContext declaration memoization." + see( `function isAsyncDeclaration` ) + InterfaceDeclaration asyncContextDeclaration = `interface AsyncTestContext`; + + "AsyncPrePostContext declaration memoization." + see( `function isAsyncPrepostDeclaration` ) + InterfaceDeclaration prepostContextDeclaration = `interface AsyncPrePostContext`; + + "AsyncFactoryContext declaration memoization." + see( `function isAsyncFactoryDeclaration` ) + InterfaceDeclaration factoryContextDeclaration = `interface AsyncFactoryContext`; + + "Cache async test executor declaration. See `function isAsyncExecutedTest`" + ClassDeclaration asyncTestDeclaration = `class AsyncTestExecutor`; + + + "`true` if execution is performed using `AsyncTestExecutor`" + shared Boolean isAsyncExecutedTest( FunctionDeclaration functionDeclaration ) => + if ( exists ann = findFirstAnnotation( functionDeclaration ) ) + then ann.executor == asyncTestDeclaration else false; + + + "Returns `true` if function takes `firstArg` as first argument." + Boolean isFirstArgSatisfies( FunctionDeclaration functionDeclaration, InterfaceDeclaration firstArg ) { + if ( is OpenInterfaceType argType = functionDeclaration.parameterDeclarations.first?.openType, + argType.declaration == firstArg + ) { + return true; + } + else { + return false; + } + } + + "Returns `true` if function runs async test == takes `AsyncTestContext` as first argument." + shared Boolean isAsyncDeclaration( FunctionDeclaration functionDeclaration ) + => isFirstArgSatisfies( functionDeclaration, asyncContextDeclaration ); + + "Returns `true` if function runs async test initialization == takes `AsyncPrePostContext` as first argument." + shared Boolean isAsyncPrepostDeclaration( FunctionDeclaration functionDeclaration ) + => isFirstArgSatisfies( functionDeclaration, prepostContextDeclaration ); + + "Returns `true` if function runs async test initialization == takes `AsyncFactoryContext` as first argument." + shared Boolean isAsyncFactoryDeclaration( FunctionDeclaration functionDeclaration ) + => isFirstArgSatisfies( functionDeclaration, factoryContextDeclaration ); + + +} diff --git a/source/herd/asynctest/extractSourceValue.ceylon b/source/herd/asynctest/internal/extractSourceValue.ceylon similarity index 55% rename from source/herd/asynctest/extractSourceValue.ceylon rename to source/herd/asynctest/internal/extractSourceValue.ceylon index 2307a98..a7b187c 100644 --- a/source/herd/asynctest/extractSourceValue.ceylon +++ b/source/herd/asynctest/internal/extractSourceValue.ceylon @@ -10,17 +10,17 @@ import ceylon.language.meta { "Calls [[source]] to get source value." since( "0.6.0" ) by( "Lis" ) -Result extractSourceValue( FunctionOrValueDeclaration source, Object? instance ) { +shared Result extractSourceValue( FunctionOrValueDeclaration source, Object? instance ) { switch ( source ) case ( is FunctionDeclaration ) { value args = resolveArgumentList( source, instance ); return if ( !source.toplevel, exists instance ) - then source.memberApply( type( instance ) ).bind( instance ).apply( *args ) - else source.apply().apply( *args ); + then source.memberApply( type( instance ) ).bind( instance ).apply( *args ) + else source.apply().apply( *args ); } case ( is ValueDeclaration ) { return if ( !source.toplevel, exists instance ) - then source.memberApply( type( instance ) ).bind( instance ).get() - else source.apply().get(); + then source.memberApply( type( instance ) ).bind( instance ).get() + else source.apply().get(); } } diff --git a/source/herd/asynctest/internal/resolveArgumentList.ceylon b/source/herd/asynctest/internal/resolveArgumentList.ceylon new file mode 100644 index 0000000..6d3c947 --- /dev/null +++ b/source/herd/asynctest/internal/resolveArgumentList.ceylon @@ -0,0 +1,27 @@ +import ceylon.language.meta.declaration { + + ClassDeclaration, + FunctionDeclaration +} +import ceylon.language.meta { + + optionalAnnotation +} +import herd.asynctest { + ArgumentsAnnotation +} + + +"Resolves argument list from `ArgumentsAnnotation`." +since( "0.5.0" ) by( "Lis" ) +shared Anything[] resolveArgumentList ( + "Declaration to resolve list" FunctionDeclaration|ClassDeclaration declaration, + "Instance of the test class or `null` if not available." Object? instance +) { + if ( exists argProvider = optionalAnnotation( `ArgumentsAnnotation`, declaration )?.source ) { + return extractSourceValue( argProvider, instance ); + } + else { + return []; + } +} diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index 4e06fbe..8370d9f 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -18,7 +18,8 @@ * [[AsyncTestExecutor]] - the test executor which satisfies `ceylon.test.engine.spi::TestExecutor` and provides an interaction with `ceylon.test` module. * [[AsyncTestContext]] which provides an interaction of the test function with the test framework. - * [[AsyncPrePostContext]] which provides an interaction of initialization / disposing logic with the test framework. + * [[AsyncPrePostContext]] which provides an interaction of initialization / disposing logic with the test framework. + * [[package herd.asynctest.parameterization]] provides value- and type- parameterized testing capability. * [[package herd.asynctest.rule]] package which contains rules used for test initialization / disposing and for modification of the test behaviour. * [[package herd.asynctest.runner]] package which provides a control over a test function execution. @@ -180,8 +181,8 @@ provider has to return the same arguments list. But async version will additionally be provided with `AsyncPrePostContext` which has to be the first argument. - > [[arguments]] annotation is not applicable to test functions. [[parameterized]] annotation is aimed - to perform parameterized testing, see, section [Value- and type- parameterized testing](#parameterized_section) below. + > [[arguments]] annotation is not applicable to test functions. + See package [[package herd.asynctest.parameterization]] for the parameterized testing. #### Test rules @@ -269,49 +270,12 @@ ------------------------------------------- ### Value- and type- parameterized testing - In order to perform parameterized testing the test function has to be marked with annotation which supports - [[TestVariantProvider]] interface. The interface has just a one method - `variants()` - which has to provide [[TestVariantEnumerator]]. The enumerator produces a stream - of the [[TestVariant]]'s and is iterated just a once. - The test will be performed using all variants the enumerator produces. + Parameterized testing is performed using annotations to specify a generator of the test variants. + Basically, each test variant is a list of test function type parameters and arguments. + Package [[package herd.asynctest.parameterization]] provides parameterized testing capability + with a number of ways to generate test variants. - > The enumerator may return test variants lazily, dynamicaly or even non-determenisticaly. - > Each [[TestVariant]] contains a list of generic type parameters and a list of function arguments. - - [[parameterized]] annotation satisfies [[TestVariantProvider]] interface and - provides parameterized testing based on collection of test variants. - - - **Custom parameterization:** - - 1. Implement [[TestVariantEnumerator]] interface: - class MyTestVariantEnumerator(...) satisfies TestVariantEnumerator { - shared actual TestVariant|Finished current => ...; - - shared actual void moveNext(TestVariantResult result) { - if (testToBeCompleted) { - // set `current` to `finished` - } else { - // set `current` to test variant to be tested next - } - } - } - - 2. Make an annotation which satisfies [[TestVariantProvider]] interface: - shared final annotation class MyParameterizedAnnotation(...) - satisfies SequencedAnnotation&TestVariantProvider - { - shared actual TestVariantEnumerator variants() => MyTestVariantEnumerator(...); - } - - shared annotation MyParameterizedAnnotation myParameterized(...) => MyParameterizedAnnotation(...); - - - 3. Mark test function with created annotation: - myParameterized(...) void myTest(...) {...} - - ------------------------------------------- ### Test runners diff --git a/source/herd/asynctest/parameterization/ArgumentVariants.ceylon b/source/herd/asynctest/parameterization/ArgumentVariants.ceylon new file mode 100644 index 0000000..3f9ed43 --- /dev/null +++ b/source/herd/asynctest/parameterization/ArgumentVariants.ceylon @@ -0,0 +1,19 @@ +import ceylon.language.meta.declaration { + FunctionOrValueDeclaration, + FunctionDeclaration +} + + +"Specifies variants for a test function argument." +tagged( "Combinatorial generators" ) +since( "0.6.1" ) by( "Lis" ) +shared final class ArgumentVariants ( + "Test function declaration." + shared FunctionDeclaration testFunction, + "The test function argument this source provides variants for." + shared FunctionOrValueDeclaration argument, + "The variants kind which indicates how to combine this argument with the others." + shared CombinatorialKind kind, + "A list of argument variants." + shared [Anything+] variants +) {} diff --git a/source/herd/asynctest/parameterization/CombinatorialKind.ceylon b/source/herd/asynctest/parameterization/CombinatorialKind.ceylon new file mode 100644 index 0000000..a588e43 --- /dev/null +++ b/source/herd/asynctest/parameterization/CombinatorialKind.ceylon @@ -0,0 +1,31 @@ + + +"Indicates the kind of the combinatorial source data." +tagged( "Kind" ) +see( `function combinatorial`, `class ArgumentVariants` ) +since( "0.6.1" ) by( "Lis" ) +shared abstract class CombinatorialKind() {} + +"Indicates that the test data has to be zipped." +tagged( "Kind" ) +since( "0.6.1" ) by( "Lis" ) +shared abstract class ZippedKind() of zippedKind extends CombinatorialKind() { + string => "zipped combinatorial kind"; +} + +"Indicates that the test data has to be zipped." +tagged( "Kind" ) +since( "0.6.1" ) by( "Lis" ) +shared object zippedKind extends ZippedKind() {} + +"Indicates that the test data has to be permuted." +tagged( "Kind" ) +since( "0.6.1" ) by( "Lis" ) +shared abstract class PermutationKind() of permutationKind extends CombinatorialKind() { + string => "permutation combinatorial kind"; +} + +"Indicates that the test data has to be permuted." +tagged( "Kind" ) +since( "0.6.1" ) by( "Lis" ) +shared object permutationKind extends PermutationKind() {} diff --git a/source/herd/asynctest/parameterization/MixingEnumerator.ceylon b/source/herd/asynctest/parameterization/MixingEnumerator.ceylon new file mode 100644 index 0000000..931244e --- /dev/null +++ b/source/herd/asynctest/parameterization/MixingEnumerator.ceylon @@ -0,0 +1,124 @@ +import herd.asynctest { + TestVariantResult +} + + +"Enumerates mixed variants." +since( "0.6.1" ) by( "Lis" ) +class MixingEnumerator( ArgumentVariants[] arguments ) + satisfies TestVariantEnumerator +{ + + "Returns first permuted kind starting from index `from`." + Integer firstPermutedKind( ArgumentVariants[] variants, Integer from ) { + variable Integer retIndex = from; + while ( exists arg = variants[retIndex], arg.kind === zippedKind ) { + retIndex ++; + } + return retIndex; + } + + Integer size = arguments.size; + "Currently looked indexes of the variants" + Array indexes = Array.ofSize( size, 0 ); + "index of current argument" + variable Integer currentArgumentIndex = firstPermutedKind( arguments, 0 ); + "indicator that mixing is still alive i.e. zip or permutation is available" + variable Boolean mixAlive = true; + + + "Shifts permuted index to next. Returns `false` if no more permutations." + Boolean shiftNextPermuted() { + if ( currentArgumentIndex < size ) { + // shift to next of permuted item + for ( i in 0 .. currentArgumentIndex ) { + assert ( exists argAtI = arguments[i] ); + // shift to next argument variant if it is of permutation kind + if ( argAtI.kind === permutationKind ) { + assert ( exists indexAtI = indexes[i] ); + if ( argAtI.variants.size > indexAtI + 1 ) { + indexes[i] = indexAtI + 1; + return true; + } + else { + // index i is full - reset it to first + indexes[i] = 0; + // shift index to 1 at next first permuted + if ( i == currentArgumentIndex ) { + // skip zipped kind + Integer startToLookNext = firstPermutedKind( arguments, currentArgumentIndex + 1 ); + // if next index is not found then permutation has to be completed + currentArgumentIndex = size; + // reset next indexes to 1 if permuted kind + for ( j in startToLookNext : size - startToLookNext ) { + if ( exists arr = arguments[j], arr.kind === permutationKind, arr.variants.size > 1 ) { + // next argument to be permuted is found + indexes[j] = 1; + currentArgumentIndex = j; + return true; + } + } + return false; + } + } + } + } + } + return false; + } + + "Shifts zip indexes to next. Returns `false` if no more zippings." + Boolean shiftNextZipped() { + if ( mixAlive ) { + variable Boolean shifted = false; + for ( i in 0 : size ) { + assert ( exists argAtI = arguments[i] ); + if ( argAtI.kind === zippedKind ) { + // shift next index for zipped kind + assert ( exists indexAtI = indexes[i] ); + if ( indexAtI < argAtI.variants.size - 1 ) { + shifted = true; + indexes[i] = indexAtI + 1; + } + else { + // all zips are exhausted - no more variants available + shifted = false; + break; + } + } + else { + // for permutation kind just reset index to zero + indexes[i] = 0; + } + } + return shifted; + } + return false; + } + + "Shifts to next element." + TestVariant|Finished shiftNext() { + if ( mixAlive ) { + value ret = TestVariant ( + [], [ for ( i in 0 : size ) getArgument( arguments[i], indexes[i] ) ] + ); + if ( !shiftNextPermuted() ) { + mixAlive = shiftNextZipped(); + currentArgumentIndex = mixAlive then firstPermutedKind( arguments, 0 ) else size; + } + return ret; + } + else { + return finished; + } + } + + variable TestVariant|Finished curVariant = shiftNext(); + + shared actual TestVariant|Finished current => curVariant; + + shared actual void moveNext( TestVariantResult result ) { + curVariant = shiftNext(); + } + +} diff --git a/source/herd/asynctest/TestVariant.ceylon b/source/herd/asynctest/parameterization/TestVariant.ceylon similarity index 84% rename from source/herd/asynctest/TestVariant.ceylon rename to source/herd/asynctest/parameterization/TestVariant.ceylon index 75e3348..c310e90 100644 --- a/source/herd/asynctest/TestVariant.ceylon +++ b/source/herd/asynctest/parameterization/TestVariant.ceylon @@ -9,7 +9,9 @@ import ceylon.language.meta.model { "Represents test variant, i.e. test function generic type parameters and arguments." -see( `interface TestVariantEnumerator`, `interface TestVariantProvider` ) +tagged( "Base" ) +see( `interface TestVariantEnumerator`, `interface TestVariantProvider`, + `class ParameterizedAnnotation`, `class CombinatorialAnnotation` ) since( "0.6.0" ) by( "Lis" ) shared class TestVariant ( "Generic type parameters." @@ -86,20 +88,3 @@ shared class TestVariant ( } } - - -"Test variant without any arguments." -since( "0.6.0" ) by( "Lis" ) -object emptyTestVariant extends TestVariant( [], [] ) { - shared actual String variantName = ""; - shared actual Boolean equals( Object that ) { - if ( is TestVariant that ) { - return that === this; - } - else { - return false; - } - } - - shared actual Integer hash => 37; -} diff --git a/source/herd/asynctest/TestVariantEnumerator.ceylon b/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon similarity index 53% rename from source/herd/asynctest/TestVariantEnumerator.ceylon rename to source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon index 8366599..2d5341e 100644 --- a/source/herd/asynctest/TestVariantEnumerator.ceylon +++ b/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon @@ -1,9 +1,13 @@ import ceylon.test { TestState } +import herd.asynctest { + TestVariantResult +} "Enumerates test variants." +tagged( "Base" ) see( `interface TestVariantProvider` ) since( "0.6.0" ) by( "Lis" ) shared interface TestVariantEnumerator { @@ -17,17 +21,7 @@ shared interface TestVariantEnumerator { } -"Provides a one empty test variant." -since( "0.6.0" ) by( "Lis" ) -class EmptyTestVariantEnumerator() satisfies TestVariantEnumerator { - variable TestVariant|Finished currentVal = emptyTestVariant; - shared actual TestVariant|Finished current => currentVal; - shared actual void moveNext(TestVariantResult result) => currentVal = finished; -} - - "Iterates variants by arguments list and completes when number of failures reach limit." -see( `function parameterized` ) since( "0.6.0" ) by( "Lis" ) class TestVariantIterator ( "Test variants iterator - function which returns next." TestVariant|Finished argsIterator(), @@ -57,41 +51,32 @@ class TestVariantIterator ( } -"Combines several test variant iterators." -since( "0.6.0" ) by( "Lis" ) -class CombinedVariantEnumerator( Iterator providers ) +"Delegates enumeration to another enumerator but completes when number of failures reach limit." +since( "0.6.1" ) by( "Lis" ) +class TestVariantMaxFailureEnumerator ( + "Test variants iterator - function which returns next." TestVariantEnumerator other, + "Limit on failures. When it is reached `next` returns `finished`." Integer maxFailedVariants +) satisfies TestVariantEnumerator { + variable Integer variantsFailed = 0; - variable TestVariantEnumerator|Finished currentProvider = providers.next(); - - TestVariant|Finished moveNextProvider() { - while ( is TestVariantEnumerator cur = currentProvider ) { - if ( is TestVariant tv = cur.current ) { - return tv; - } - else { - currentProvider = providers.next(); - } + shared actual TestVariant|Finished current { + if ( maxFailedVariants < 1 || maxFailedVariants > variantsFailed ) { + return other.current; + } + else { + return finished; } - return finished; } - variable TestVariant|Finished currrentVariant = moveNextProvider(); - - - shared actual TestVariant|Finished current => currrentVariant; - shared actual void moveNext( TestVariantResult result ) { - if ( is TestVariantEnumerator cur = currentProvider ) { - cur.moveNext( result ); - if ( is TestVariant tv = cur.current ) { - currrentVariant = tv; - return; - } + if ( result.overallState > TestState.success ) { + variantsFailed ++; + } + if ( maxFailedVariants < 1 || maxFailedVariants > variantsFailed ) { + other.moveNext( result ); } - currrentVariant = moveNextProvider(); - } + } } - diff --git a/source/herd/asynctest/TestVariantProvider.ceylon b/source/herd/asynctest/parameterization/TestVariantProvider.ceylon similarity index 91% rename from source/herd/asynctest/TestVariantProvider.ceylon rename to source/herd/asynctest/parameterization/TestVariantProvider.ceylon index bbe2caf..21efdfb 100644 --- a/source/herd/asynctest/TestVariantProvider.ceylon +++ b/source/herd/asynctest/parameterization/TestVariantProvider.ceylon @@ -6,8 +6,9 @@ import ceylon.language.meta.declaration { "Provides enumerator for test variants. Test execution context looks for annotations of the test function which support the interface and performs testing according to provided variants. - As example, see [[parameterized]]. + As example, see [[ParameterizedAnnotation]]. " +tagged( "Base" ) see( `class TestVariant` ) since( "0.6.0" ) by( "Lis" ) shared interface TestVariantProvider { diff --git a/source/herd/asynctest/parameterization/combinatorial.ceylon b/source/herd/asynctest/parameterization/combinatorial.ceylon new file mode 100644 index 0000000..06f99e7 --- /dev/null +++ b/source/herd/asynctest/parameterization/combinatorial.ceylon @@ -0,0 +1,94 @@ +import ceylon.language.meta.declaration { + FunctionDeclaration, + FunctionOrValueDeclaration +} +import ceylon.language.meta { + type, + optionalAnnotation +} +import herd.asynctest.internal { + declarationVerifier, + extractSourceValue +} + + +"Indicates that the test function has to be executed with variants provided by combinations + of the test function agrument values. + [[DataSourceAnnotation]] annotation has to be applied to the each argument of the test function in order to + specify a list of possible argument values. + [[combinator]] function is responsible to generate test variants and to provide [[TestVariantEnumerator]]. + " +tagged( "Applying combinatorial" ) +see( `function combinatorial`, `class ParameterizedAnnotation`, `class DataSourceAnnotation`, + `function permuting`, `function zipping`, `function mixing` ) +since( "0.6.1" ) by( "Lis" ) +shared final annotation class CombinatorialAnnotation ( + "Combinator function which has to return [[TestVariantEnumerator]] and takes `ArgumentVariants[]` + which is generated based on [[DataSourceAnnotation]] at each test function argument. + > Each test function argument has to be marked with [[DataSourceAnnotation]]!" + shared FunctionDeclaration combinator, + "Maximum number of failed variants to stop testing. Unlimited if <= 0." + shared Integer maxFailedVariants +) + satisfies SequencedAnnotation & TestVariantProvider +{ + + "Asserts if argument list is empty." + [Anything+] extractNonemptyArgumentList( FunctionOrValueDeclaration source, Object? instance ) { + "Agrument list for combinatorial testing has to be nonempty." + assert( nonempty ret = extractSourceValue( source, instance ) ); + return ret; + } + + + "Extracts argument source if available." + ArgumentVariants resolveArgumentSource ( + "Test function to extract source for." FunctionDeclaration testFunction, + "Declaration to resolve argument source." FunctionOrValueDeclaration declaration, + "Instance of the test class or `null` if not available." Object? instance + ) { + if ( exists argProvider = optionalAnnotation( `DataSourceAnnotation`, declaration ) ) { + return ArgumentVariants ( + testFunction, declaration, + extractSourceValue( argProvider.kind, instance ), + extractNonemptyArgumentList( argProvider.source, instance ) + ); + } + else { + throw AssertionError( "Argument ``declaration.name`` of test function ``testFunction.name`` doesn't have data source." ); + } + } + + + throws( `class AssertionError`, "At least one of the test function argument doesn't contain `argument` annotation + or argument source function returns empty argument list." ) + shared actual TestVariantEnumerator variants( FunctionDeclaration testFunction, Object? instance ) { + value params = + if ( declarationVerifier.isAsyncDeclaration( testFunction ) ) + then testFunction.parameterDeclarations.rest + else testFunction.parameterDeclarations; + value args = [ for ( item in params ) resolveArgumentSource( testFunction, item, instance ) ]; + return TestVariantMaxFailureEnumerator ( + if ( !combinator.toplevel, exists instance ) + then combinator.memberApply( type( instance ) ).bind( instance ).apply( args ) + else combinator.apply().apply( args ), + maxFailedVariants + ); + } + +} + + +"Indicates the test has to be performed with combinatorial test variants. + See details in [[CombinatorialAnnotation]]." +tagged( "Applying combinatorial" ) +see( `function permuting`, `function zipping`, `function mixing` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation CombinatorialAnnotation combinatorial ( + "Combinator function which has to return [[TestVariantEnumerator]] and takes `ArgumentVariants[]` + which is generated based on [[DataSourceAnnotation]] at each test function argument. + > Each test function argument has to be marked with [[DataSourceAnnotation]]!" + FunctionDeclaration combinator, + "Maximum number of failed variants to stop testing. Unlimited if <= 0." + Integer maxFailedVariants = -1 +) => CombinatorialAnnotation( combinator, maxFailedVariants ); diff --git a/source/herd/asynctest/parameterization/dataSource.ceylon b/source/herd/asynctest/parameterization/dataSource.ceylon new file mode 100644 index 0000000..a535257 --- /dev/null +++ b/source/herd/asynctest/parameterization/dataSource.ceylon @@ -0,0 +1,61 @@ +import ceylon.language.meta.declaration { + FunctionOrValueDeclaration, + ValueDeclaration +} + + +"Indicates that the test has to be performed by applying a list of values to the marked test function argument. + [[source]] has to provide the values. + [[kind]] indicates the kind of the provided values. + The kind is used by _variant generator_ in order to identify strategy for the variant generations." +tagged( "Data source" ) +see( `class CombinatorialAnnotation` ) +since( "0.6.1" ) by( "Lis" ) +shared final annotation class DataSourceAnnotation ( + "The source function or value declaration which has to return a stream of values. + The source may be either top-level or tested class shared member." + shared FunctionOrValueDeclaration source, + "The test data kind. Has to extend [[CombinatorialKind]]." + shared ValueDeclaration kind +) + satisfies OptionalAnnotation +{} + + +"Provides values for the marked test function argument. See [[DataSourceAnnotation]] for details." +tagged( "Data source" ) +see( `function zippedSource`, `function permutationSource` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation DataSourceAnnotation dataSource ( + "The source function or value declaration which has to return a stream of values. + The source may be either top-level or tested class shared member." + FunctionOrValueDeclaration source, + "The test data kind. Returned value has to extend [[CombinatorialKind]]." + ValueDeclaration kind +) => DataSourceAnnotation( source, kind ); + + +"Provides [[zippedKind]] kind values for the marked test function argument. + See [[DataSourceAnnotation]] for details. + The annotation is shortcut for `\`dataSource(`value zippedKind`)`\`." +tagged( "Data source" ) +see( `function dataSource`, `function permutationSource` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation DataSourceAnnotation zippedSource ( + "The source function or value declaration which has to return a stream of values. + The source may be either top-level or tested class shared member." + FunctionOrValueDeclaration source +) => DataSourceAnnotation( source, `value zippedKind` ); + + +"Provides [[permutationKind]] kind values for the marked test function argument. + See [[DataSourceAnnotation]] for details. + The annotation is shortcut for `\`dataSource(`value permutationKind`)`\`." +tagged( "Data source" ) +see( `function zippedSource`, `function dataSource` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation DataSourceAnnotation permutationSource ( + "The source function or value declaration which has to return a stream of values. + The source may be either top-level or tested class shared member." + FunctionOrValueDeclaration source +) => DataSourceAnnotation( source, `value permutationKind` ); diff --git a/source/herd/asynctest/parameterization/mixingCombinations.ceylon b/source/herd/asynctest/parameterization/mixingCombinations.ceylon new file mode 100644 index 0000000..0c590f0 --- /dev/null +++ b/source/herd/asynctest/parameterization/mixingCombinations.ceylon @@ -0,0 +1,72 @@ + +"Returns [[CombinatorialAnnotation]] with [[mixingCombinations]] combinator function. + I.e. it is shortcut for `\`combinatorial(`function mixingCombinations`)`\`. + + Note: + * [[zippedSource]] provides variants of [[ZippedKind]]. + * [[permutationSource]] provides variants of [[PermutationKind]]. + + Example: + Integer[] firstArgument => [1,2]; + Integer[] secondArgument => [10,20]; + String[] signArgument => [\"+\",\"-\"]; + + async test mixing void testMixing ( + zippedSource(\`value firstArgument\`) Integer arg1, + permutationSource(\`value signArgument\`) String arg2, + zippedSource(\`value secondArgument\`) Integer arg3 + ) {...} + + In the above example: + * [[mixing]] annotation forces to use `mixingCombinations` as variant generator. + * First and third argument are zipped. + * Second argument is permuted. + + As result four test variants are generated: + 1. `arg1` = 1, `arg2` = \"+\", `arg3` = 10. + 2. `arg1` = 1, `arg2` = \"-\", `arg3` = 10. + 3. `arg1` = 2, `arg2` = \"+\", `arg3` = 20. + 4. `arg1` = 2, `arg2` = \"-\", `arg3` = 20. + + " +tagged( "Applying combinatorial" ) +see( `function combinatorial`, `function mixingCombinations`, `function permutationSource`, `function zippedSource`, + `function permuting`, `function zipping` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation CombinatorialAnnotation mixing ( + "Maximum number of failed variants to stop testing. Unlimited if <= 0." + Integer maxFailedVariants = -1 +) => CombinatorialAnnotation( `function mixingCombinations`, maxFailedVariants ); + + +"Returns first permuted kind starting from index `from`." +since( "0.6.1" ) by( "Lis" ) +Integer firstPermutedKind( ArgumentVariants[] variants, Integer from ) { + variable Integer retIndex = from; + while ( exists arg = variants[retIndex ++], arg.kind === zippedKind ) {} + return retIndex; +} + + +"Mixes zip and permutation sources. I.e. some arguments may be provided with [[zippedSource]] + and the other arguments are provided with [[permutationSource]]. + + Generation rule: for each zipped variant every possible permutations are generated. + + See example in [[mixing]]. + " +tagged( "Combinatorial generators" ) +see( `function mixing`, `function permutationSource`, `function zippedSource` ) +since( "0.6.1" ) by( "Lis" ) +shared TestVariantEnumerator mixingCombinations ( + "Variants of the test function arguments." ArgumentVariants[] arguments +) { + // verify each argument is permutation kind + for ( item in arguments ) { + if ( !( item.kind === permutationKind || item.kind === zippedKind ) ) { + throw AssertionError( "To perform mix combination argument ``item.argument.name`` of test function ``item.testFunction.name`` has to be 'zipped' or 'permutation' kind" ); + } + } + + return MixingEnumerator( arguments ); +} diff --git a/source/herd/asynctest/parameterization/package.ceylon b/source/herd/asynctest/parameterization/package.ceylon new file mode 100644 index 0000000..f1fcf26 --- /dev/null +++ b/source/herd/asynctest/parameterization/package.ceylon @@ -0,0 +1,120 @@ + +" + Value- and type- parameterized testing. + + + ### Base + + In order to perform parameterized testing the test function has to be marked with annotation which supports + [[TestVariantProvider]] interface. The interface has just a one method - `variants()` + which has to provide [[TestVariantEnumerator]]. The enumerator produces a stream + of the [[TestVariant]]'s and is iterated just a once. + The test will be performed using all variants the enumerator produces. + + > The enumerator may return test variants lazily, dynamicaly or even non-determenisticaly. + > Each [[TestVariant]] contains a list of generic type parameters and a list of function arguments. + + + ### Custom parameterization + + 1. Implement [[TestVariantEnumerator]] interface: + class MyTestVariantEnumerator(...) satisfies TestVariantEnumerator { + shared actual TestVariant|Finished current => ...; + + shared actual void moveNext(TestVariantResult result) { + if (testToBeCompleted) { + // set `current` to `finished` + } else { + // set `current` to test variant to be tested next + } + } + } + + 2. Make an annotation which satisfies [[TestVariantProvider]] interface: + shared final annotation class MyParameterizedAnnotation(...) + satisfies SequencedAnnotation&TestVariantProvider + { + shared actual TestVariantEnumerator variants() => MyTestVariantEnumerator(...); + } + + shared annotation MyParameterizedAnnotation myParameterized(...) => MyParameterizedAnnotation(...); + + + 3. Mark test function with created annotation: + myParameterized(...) void myTest(...) {...} + + + ### Varianted testing + + [[parameterized]] annotation satisfies [[TestVariantProvider]] interface and + provides type- and value- parameterized testing based on collection of test variants. + + + ### Combinatorial testing + + Combinatorial testing provides generation of test variants based on data sources of each test function argument. + + #### Therminology + + * _Data source_ is a list of the values to be applied as particular value of the test function argument. + * _Variant generator_ is a function which generates test variants based on _data sources_ for each test function argument. + * _Argument kind_ is an indicator which is applied to each argument of test function + and which _variant generator_ may used to identify particular strategy of the variants generation. + + #### Usage + + 1. Declare test function: + void combinatorialTest(Something arg1, Something arg2); + 2. Declare _data sources_. + Something[] arg1Source => + Something[] arg2Source => + 3. Apply _data source_ for each argument of the test function. + void combinatorialTest(permutationSource(`arg1Source`) Something arg1, + permutationSource(`arg2Source`) Something arg2); + 4. Apply _variant generator_ to test function + permuting void combinatorialTest(permutationSource(`arg1Source`) Something arg1, + permutationSource(`arg2Source`) Something arg2); + 5. Mark the function with `async` and `test` annotations and run test + async test permuting void combinatorialTest(permutationSource(`arg1Source`) Something arg1, + permutationSource(`arg2Source`) Something arg2); + + #### Argument kind + + The kind is used by _variant generator_ in order to identify strategy for the variant generations. + There are two kinds: + * [[ZippedKind]] indicates that the data has to be combined with others by zipping the sources, + i.e. combine the values with the same index. + * [[PermutationKind]] indicates that the data has to be combined with others + using all possible permutations of the source. + + Custom kind has to extend [[CombinatorialKind]]. + + #### Data source + + [[DataSourceAnnotation]] provides for the marked argument of the test function: + * list of the argument values + * argument kind + + There are three functions to apply the annotation: + 1. [[dataSource]] which instantiates [[DataSourceAnnotation]] using both parameters. + 2. [[zippedSource]] which provides a list of the argument values and mark argument as [[zippedKind]]. + 3. [[permutationSource]] which provides a list of the argument values and mark argument as [[permutationKind]]. + + > [[DataSourceAnnotation]] annotation has to be applied to the each argument of the test function. + + #### Variant generator + + Is applied to a test function with [[CombinatorialAnnotation]] annotation. + There are four functions to apply the annotation: + * [[combinatorial]] applies custom _variant generator_ + * [[zipping]] applies [[zippingCombinations]] _variant generator_ which zips the arguments + (each argument has to be [[zippedKind]]) + * [[permuting]] applies [[permutingCombinations]] _variant generator_ which permutes all possible variants + (each argument has to be [[permutationKind]]) + * [[mixing]] applies [[mixingCombinations]] _variant generator_ which mixes zipped and permuted arguments + + > Custom _variant generator_ has to take a list of [[ArgumentVariants]] and has to return [[TestVariantEnumerator]]. + +" +since( "0.6.1" ) by( "Lis" ) +shared package herd.asynctest.parameterization; diff --git a/source/herd/asynctest/parameterization/parameterized.ceylon b/source/herd/asynctest/parameterization/parameterized.ceylon new file mode 100644 index 0000000..b2d11a9 --- /dev/null +++ b/source/herd/asynctest/parameterization/parameterized.ceylon @@ -0,0 +1,110 @@ +import herd.asynctest.internal { + extractSourceValue +} +import ceylon.language.meta.declaration { + FunctionOrValueDeclaration, + FunctionDeclaration +} + + +"Indicates that generic (with possibly empty generic parameter list) test function + has to be executed with given test variants. + + The annotation provides parameterized testing based on collection of test variants. + It takes two arguments: + 1. Declaration of function or value which returns a collection of test variants `{TestVariant*}`. + 2. Number of failed variants to stop testing. Default is -1 which means no limit. + + The test will be performed using all test variants returned by the given stream + or while total number of failed variants not exceeds specified limit. + + > [[parameterized]] annotation may occur multiple times at a given test function. + > The variants source may be either top-level or tested class shared member. + + + #### Example: + + Value identity(Value argument) => argument; + + {TestVariant*} identityArgs => { + TestVariant([`String`], [\"stringIdentity\"]), + TestVariant([`Integer`], [1]), + TestVariant([`Float`], [1.0]) + }; + + shared test async + parameterized(`value identityArgs`) + void testIdentity(AsyncTestContext context, Value arg) + given Value satisfies Object + { + context.assertThat(identity(arg), EqualObjects(arg), \"\", true); + context.complete(); + } + + In the above example the function `testIdentity` will be called 3 times: + * `testIdentity(context, \"stringIdentity\");` + * `testIdentity(context, 1);` + * `testIdentity(context, 1.0);` + + In order to run test with conventional (non-generic function) type parameters list has to be empty: + [Hobbit] who => [bilbo]; + {TestVariant*} dwarves => { + TestVariant([], [fili]), + TestVariant([], [kili]), + TestVariant([], [balin], + TestVariant([], [dwalin]), + ... + }; + + arguments(`value who`) + class HobbitTester(Hobbit hobbit) { + shared test async + parameterized(`value dwarves`, 2) + void thereAndBackAgain(AsyncTestContext context, Dwarf dwarf) { + context.assertTrue(hobbit.thereAndBackAgain(dwarf)...); + context.complete(); + } + } + + In this example class `HobbitTester` is instantiated once with argument provided by value `who` and + method `thereAndBackAgain` is called multiply times according to size of `dwarves` stream. + According to second argument of `parameterized` annotation the test will be stopped + if two different invoking of `thereAndBackAgain` with two different arguments report failure. + " +tagged( "varianted" ) +see( `function parameterized`, `class TestVariant`, `class CombinatorialAnnotation` ) +since( "0.6.0" ) by( "Lis" ) +shared final annotation class ParameterizedAnnotation ( + "The source function or value declaration which has to return a stream + of test variants: `{TestVariant*}`. + The source may be either top-level or tested class shared member." + shared FunctionOrValueDeclaration source, + "Maximum number of failed variants to stop testing. Unlimited if <= 0." + shared Integer maxFailedVariants +) + satisfies SequencedAnnotation & TestVariantProvider +{ + + "Returns test variant enumerator based on test variants extracted from `source`." + shared actual TestVariantEnumerator variants( FunctionDeclaration testFunction, Object? instance) + => TestVariantIterator ( + extractSourceValue<{TestVariant*}>( source, instance ).iterator().next, + maxFailedVariants + ); + +} + + +"Provides parameters for the parameterized testing. See [[ParameterizedAnnotation]] for the details." +tagged( "varianted" ) +see( `class TestVariant` ) +since( "0.6.0" ) by( "Lis" ) +shared annotation ParameterizedAnnotation parameterized ( + "The source function or value declaration which has to take no arguments and has to return + a stream of test variants: `{TestVariant*}`. + The source may be either top-level or tested class shared member." + FunctionOrValueDeclaration source, + "Maximum number of failed variants before stop. Unlimited if <= 0." + Integer maxFailedVariants = -1 +) + => ParameterizedAnnotation( source, maxFailedVariants ); diff --git a/source/herd/asynctest/parameterization/permutingCombinations.ceylon b/source/herd/asynctest/parameterization/permutingCombinations.ceylon new file mode 100644 index 0000000..3aa7450 --- /dev/null +++ b/source/herd/asynctest/parameterization/permutingCombinations.ceylon @@ -0,0 +1,102 @@ + + +"Returns [[CombinatorialAnnotation]] with [[permutingCombinations]] combinator function. + I.e. it is shortcut for `\`combinatorial(`function permutingCombinations`)`\`. + + Example: + + Integer[] firstArgument => [1,2]; + Integer[] secondArgument => [10,20,30]; + + async test permuting void myTest ( + permutationSource(\`value firstArgument\`) Integer arg1, + permutationSource(\`value secondArgument\`) Integer arg2 + ) {...} + + The test will be performed using six test variants: + 1. `arg1` = 1, `arg2` = 10 + 2. `arg1` = 1, `arg2` = 20 + 3. `arg1` = 1, `arg2` = 30 + 4. `arg1` = 2, `arg2` = 10 + 5. `arg1` = 2, `arg2` = 20 + 6. `arg1` = 2, `arg2` = 30 + " +tagged( "Applying combinatorial" ) +see( `function combinatorial`, `function permutingCombinations`, `function permutationSource`, + `function zipping`, `function mixing` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation CombinatorialAnnotation permuting ( + "Maximum number of failed variants to stop testing. Unlimited if <= 0." + Integer maxFailedVariants = -1 +) => CombinatorialAnnotation( `function permutingCombinations`, maxFailedVariants ); + + +"Extracts argument by asserting existing." +since( "0.6.1" ) by( "Lis" ) +Anything getArgument( ArgumentVariants? args, Integer? index ) { + assert ( exists args ); + assert ( exists index ); + return args.variants[index]; +} + + +"Enumerates test variants by all possible combinations from provided `arguments` + of `ArgumentVariants[]` which provides list of variants for the test function arguments. + Each function argument has to be of [[PermutationKind]] kind. + + Note: [[permutationSource]] provides variants of [[PermutationKind]]. + + See example in [[permuting]]." +tagged( "Combinatorial generators" ) +see( `function permuting`, `function permutationSource` ) +since( "0.6.1" ) by( "Lis" ) +shared TestVariantEnumerator permutingCombinations ( + "Variants of the test function arguments." ArgumentVariants[] arguments +) { + // verify each argument is permutation kind + for ( item in arguments ) { + if ( !( item.kind === permutationKind ) ) { + throw AssertionError( "To perform permutation combination argument ``item.argument.name`` of test function ``item.testFunction.name`` has to be 'permutation' kind" ); + } + } + // variants enumerator + Integer size = arguments.size; + Array indexes = Array.ofSize( size, 0 ); + variable Integer columnIndex = 0; + return TestVariantIterator ( + () { + if ( columnIndex < size ) { + value ret = TestVariant ( + [], [ for ( i in 0 : size ) getArgument( arguments[i], indexes[i] ) ] + ); + for ( i in 0 .. columnIndex ) { + assert ( exists currentIndex = indexes[i] ); + assert ( exists currentArg = arguments[i] ); + if ( currentArg.variants.size > currentIndex + 1 ) { + indexes[i] = currentIndex + 1; + break; + } + else { + indexes[i] = 0; + if ( i == columnIndex ) { + columnIndex ++; + for ( j in columnIndex : size - columnIndex ) { + if ( exists arr = arguments[j], arr.variants.size > 1 ) { + indexes[j] = 1; + break; + } + columnIndex ++; + } + break; + } + } + } + return ret; + } + else { + return finished; + } + }, + -1 + ); +} diff --git a/source/herd/asynctest/parameterization/zippingCombinations.ceylon b/source/herd/asynctest/parameterization/zippingCombinations.ceylon new file mode 100644 index 0000000..e8045e3 --- /dev/null +++ b/source/herd/asynctest/parameterization/zippingCombinations.ceylon @@ -0,0 +1,76 @@ + + +"Returns [[CombinatorialAnnotation]] with [[zippingCombinations]] combinator function. + I.e. it is shortcut for `\`combinatorial(`function zippingCombinations`)`\`. + + Example: + + Integer[] firstArgument => [1,2,3]; + Integer[] secondArgument => [10,20,30]; + + async test zipping void myTest ( + zippedSource(\`value firstArgument\`) Integer arg1, + zippedSource(\`value secondArgument\`) Integer arg2 + ) {...} + + The test will be performed using three test variants: + 1. `arg1` = 1, `arg2` = 10 + 2. `arg1` = 2, `arg2` = 20 + 3. `arg1` = 3, `arg2` = 30 + + " +tagged( "Applying combinatorial" ) +see( `function combinatorial`, `function zippingCombinations`, `function zippedSource`, + `function permuting`, `function mixing` ) +since( "0.6.1" ) by( "Lis" ) +shared annotation CombinatorialAnnotation zipping ( + "Maximum number of failed variants to stop testing. Unlimited if <= 0." + Integer maxFailedVariants = -1 +) => CombinatorialAnnotation( `function zippingCombinations`, maxFailedVariants ); + + +"Indicates that iterator finished." +since( "0.6.1" ) by( "Lis" ) +abstract class FinishedIndicator() of finishedIndicator {} +object finishedIndicator extends FinishedIndicator() {} + + +"Enumerates test variants by zipping provided `arguments` + of `ArgumentVariants[]` which provides list of variants for the test function arguments. + Each function argument has to be of [[ZippedKind]] kind. + + Note: [[zippedSource]] provides variants of [[ZippedKind]]. + + Zipping means that 'n-th' variant is built from each 'n-th' item of the argument lists. + The variant list finished at the same time when a one of argument lists finishes. + If the other lists contain more items then those items are simply ignored. + + See example in [[zipping]]. + " +tagged( "Combinatorial generators" ) +see( `function zipping`, `function zippedSource` ) +since( "0.6.1" ) by( "Lis" ) +shared TestVariantEnumerator zippingCombinations ( + "Variants of the test function arguments." ArgumentVariants[] arguments +) { + // verify that every argument is zipped combinatorial kind + for ( item in arguments ) { + if ( !( item.kind === zippedKind ) ) { + throw AssertionError( "To perform zip combination argument ``item.argument.name`` of test function ``item.testFunction.name`` has to be 'zipped' kind" ); + } + } + // collect data and instantiate enumerator + value streams = [ for ( item in arguments ) item.variants.iterator() ]; + return TestVariantIterator ( + () { + value ret = [ for ( item in streams ) let ( n = item.next() ) if ( is Finished n ) then finishedIndicator else n ]; + if ( ret.narrow().empty ) { + return TestVariant( [], ret ); + } + else { + return finished; + } + }, + -1 + ); +} diff --git a/source/herd/asynctest/resolveArguments.ceylon b/source/herd/asynctest/resolveArguments.ceylon deleted file mode 100644 index da0f1b2..0000000 --- a/source/herd/asynctest/resolveArguments.ceylon +++ /dev/null @@ -1,43 +0,0 @@ -import ceylon.language.meta.declaration { - - FunctionDeclaration, - ClassDeclaration -} -import ceylon.language.meta { - - optionalAnnotation -} - - -"Resolves a list of type parameters and function arguments provided by [[parameterized]] annotation. - Returns a list of parameters list." -since( "0.3.0" ) by( "Lis" ) -TestVariantEnumerator resolveParameterizedList ( - "Function to resolve arguments for." FunctionDeclaration declaration, - "Instance of the test class or `null` if test is performed using top-level function." Object? instance -) { - value providers = declaration.annotations().narrow(); - if ( providers.empty ) { - return EmptyTestVariantEnumerator(); - } - else if ( providers.size == 1 ) { - return providers.first?.variants( declaration, instance ) else EmptyTestVariantEnumerator(); - } - else { - return CombinedVariantEnumerator( providers*.variants( declaration, instance ).iterator() ); - } -} - -"Resolves argument list from `ArgumentsAnnotation`." -since( "0.5.0" ) by( "Lis" ) -Anything[] resolveArgumentList ( - "Declaration to resolve list" FunctionDeclaration|ClassDeclaration declaration, - "Instance of the test class or `null` if not available." Object? instance -) { - if ( exists argProvider = optionalAnnotation( `ArgumentsAnnotation`, declaration )?.source ) { - return extractSourceValue( argProvider, instance ); - } - else { - return []; - } -} diff --git a/source/herd/asynctest/variantEnumerators.ceylon b/source/herd/asynctest/variantEnumerators.ceylon new file mode 100644 index 0000000..e2317a5 --- /dev/null +++ b/source/herd/asynctest/variantEnumerators.ceylon @@ -0,0 +1,70 @@ +import herd.asynctest.parameterization { + TestVariant, + TestVariantEnumerator +} + + + +"Test variant without any arguments." +since( "0.6.0" ) by( "Lis" ) +shared object emptyTestVariant extends TestVariant( [], [] ) { + shared actual String variantName = ""; + shared actual Boolean equals( Object that ) { + if ( is TestVariant that ) { + return that === this; + } + else { + return false; + } + } + + shared actual Integer hash => 37; +} + + +"Provides a one empty test variant." +since( "0.6.0" ) by( "Lis" ) +shared class EmptyTestVariantEnumerator() satisfies TestVariantEnumerator { + variable TestVariant|Finished currentVal = emptyTestVariant; + shared actual TestVariant|Finished current => currentVal; + shared actual void moveNext(TestVariantResult result) => currentVal = finished; +} + + +"Combines several test variant iterators." +since( "0.6.0" ) by( "Lis" ) +shared class CombinedVariantEnumerator( Iterator providers ) + satisfies TestVariantEnumerator +{ + + variable TestVariantEnumerator|Finished currentProvider = providers.next(); + + TestVariant|Finished moveNextProvider() { + while ( is TestVariantEnumerator cur = currentProvider ) { + if ( is TestVariant tv = cur.current ) { + return tv; + } + else { + currentProvider = providers.next(); + } + } + return finished; + } + + variable TestVariant|Finished currrentVariant = moveNextProvider(); + + + shared actual TestVariant|Finished current => currrentVariant; + + shared actual void moveNext( TestVariantResult result ) { + if ( is TestVariantEnumerator cur = currentProvider ) { + cur.moveNext( result ); + if ( is TestVariant tv = cur.current ) { + currrentVariant = tv; + return; + } + } + currrentVariant = moveNextProvider(); + } + +} From 222d69255505e3a6963bac48ff43cffe00b04d18 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 19 Dec 2016 19:03:15 +0300 Subject: [PATCH 50/72] repackaging --- .../herd/asynctest/AsyncPrePostContext.ceylon | 3 ++ .../herd/asynctest/AsyncTestProcessor.ceylon | 8 ++-- ...stOutput.ceylon => ExecutionOutput.ceylon} | 41 ++++--------------- source/herd/asynctest/FunctionWrappers.ceylon | 3 ++ source/herd/asynctest/PrePostContext.ceylon | 6 +++ .../herd/asynctest/TestGroupExecutor.ceylon | 4 ++ source/herd/asynctest/Tester.ceylon | 8 +++- source/herd/asynctest/annotations.ceylon | 2 +- source/herd/asynctest/chart/package.ceylon | 2 + .../evaluateAnnotatedConditions.ceylon | 4 ++ source/herd/asynctest/exceptions.ceylon | 3 ++ source/herd/asynctest/match/package.ceylon | 2 + .../parameterization/MixingEnumerator.ceylon | 5 +-- .../parameterization/TestOutput.ceylon | 31 ++++++++++++++ .../TestVariantEnumerator.ceylon | 3 -- .../parameterization/combinatorial.ceylon | 3 +- .../asynctest/parameterization/package.ceylon | 5 ++- .../rule/ChainedPrePostContext.ceylon | 4 +- .../asynctest/runner/AsyncTestRunner.ceylon | 4 +- .../asynctest/runner/ChainedRunner.ceylon | 4 +- .../asynctest/runner/CollectorContext.ceylon | 3 +- .../runner/ErrorCollectorRunner.ceylon | 4 +- .../herd/asynctest/runner/RepeatRunner.ceylon | 5 ++- .../asynctest/runner/RepeatStrategy.ceylon | 9 ++-- .../asynctest/{ => runner}/TestInfo.ceylon | 7 ++-- .../{ => runner}/VariantResultBuilder.ceylon | 6 ++- .../herd/asynctest/variantEnumerators.ceylon | 9 ++-- 27 files changed, 115 insertions(+), 73 deletions(-) rename source/herd/asynctest/{TestOutput.ceylon => ExecutionOutput.ceylon} (58%) create mode 100644 source/herd/asynctest/parameterization/TestOutput.ceylon rename source/herd/asynctest/{ => runner}/TestInfo.ceylon (96%) rename source/herd/asynctest/{ => runner}/VariantResultBuilder.ceylon (94%) diff --git a/source/herd/asynctest/AsyncPrePostContext.ceylon b/source/herd/asynctest/AsyncPrePostContext.ceylon index 2e3296e..48c429a 100644 --- a/source/herd/asynctest/AsyncPrePostContext.ceylon +++ b/source/herd/asynctest/AsyncPrePostContext.ceylon @@ -3,6 +3,9 @@ import herd.asynctest.rule { SuiteRule, TestRule } +import herd.asynctest.runner { + TestInfo +} "Allows prepost function functions to interract with test executor. diff --git a/source/herd/asynctest/AsyncTestProcessor.ceylon b/source/herd/asynctest/AsyncTestProcessor.ceylon index 1bd5de0..36ddd77 100644 --- a/source/herd/asynctest/AsyncTestProcessor.ceylon +++ b/source/herd/asynctest/AsyncTestProcessor.ceylon @@ -35,16 +35,18 @@ import herd.asynctest.internal { declarationVerifier } import herd.asynctest.runner { - AsyncTestRunner, RunWithAnnotation, RepeatStrategy, - repeatOnce + repeatOnce, + TestInfo } import herd.asynctest.parameterization { TestVariantEnumerator, TestVariant, - TestVariantProvider + TestVariantProvider, + TestOutput, + TestVariantResult } diff --git a/source/herd/asynctest/TestOutput.ceylon b/source/herd/asynctest/ExecutionOutput.ceylon similarity index 58% rename from source/herd/asynctest/TestOutput.ceylon rename to source/herd/asynctest/ExecutionOutput.ceylon index 4c78070..7871aa6 100644 --- a/source/herd/asynctest/TestOutput.ceylon +++ b/source/herd/asynctest/ExecutionOutput.ceylon @@ -1,38 +1,13 @@ -import ceylon.test { - - TestState -} import ceylon.test.engine.spi { - TestExecutionContext } -import herd.asynctest.parameterization { - TestVariantEnumerator -} - - -"Represents a one report." -see( `class TestVariantResult` ) -since( "0.0.1" ) by( "Lis" ) -shared final class TestOutput ( - "Test state of this report." shared TestState state, - "Error if occured." shared Throwable? error, - "Time in milliseconds elapsed before reporting." shared Integer elapsedTime, - "Output title." shared String title -) { - string => "``state``: ``title``"; +import ceylon.test { + TestState } - -"Results of a one test function run with some arguments." -see( `interface TestVariantEnumerator`, `class VariantResultBuilder` ) -since( "0.6.0" ) by( "Lis" ) -shared final class TestVariantResult ( - "Outputs from test." shared TestOutput[] testOutput, - "Overall time in milliseconds elapsed on the test run." shared Integer overallElapsedTime, - "Overall state of the test run." shared TestState overallState -) { - string => "``overallState`` at ``overallElapsedTime``ms."; +import herd.asynctest.parameterization { + TestOutput, + TestVariantResult } @@ -48,9 +23,9 @@ final class VariantTestOutput ( ) { shared TestVariantResult variantResult => TestVariantResult ( - initOutput.append( testOutput ).append( disposeOutput ), - totalElapsedTime, totalState - ); + initOutput.append( testOutput ).append( disposeOutput ), + totalElapsedTime, totalState + ); "`true` if `initOutput`, `testOutput` and `disposeOutput` all together are empty and `false` otherwise." shared Boolean emptyOutput => initOutput.empty && testOutput.empty && disposeOutput.empty; diff --git a/source/herd/asynctest/FunctionWrappers.ceylon b/source/herd/asynctest/FunctionWrappers.ceylon index 10749a0..96f5ff6 100644 --- a/source/herd/asynctest/FunctionWrappers.ceylon +++ b/source/herd/asynctest/FunctionWrappers.ceylon @@ -1,6 +1,9 @@ import ceylon.language.meta.declaration { FunctionDeclaration } +import herd.asynctest.runner { + TestInfo +} "Function which run before or after test." diff --git a/source/herd/asynctest/PrePostContext.ceylon b/source/herd/asynctest/PrePostContext.ceylon index b3bb768..2858d1c 100644 --- a/source/herd/asynctest/PrePostContext.ceylon +++ b/source/herd/asynctest/PrePostContext.ceylon @@ -19,6 +19,12 @@ import java.lang { Thread } +import herd.asynctest.runner { + TestInfo +} +import herd.asynctest.parameterization { + TestOutput +} "Performs initialization or disposing. diff --git a/source/herd/asynctest/TestGroupExecutor.ceylon b/source/herd/asynctest/TestGroupExecutor.ceylon index 45f8abb..b151925 100644 --- a/source/herd/asynctest/TestGroupExecutor.ceylon +++ b/source/herd/asynctest/TestGroupExecutor.ceylon @@ -61,6 +61,10 @@ import java.util.concurrent.locks { ReentrantLock } +import herd.asynctest.parameterization { + TestOutput +} + "Posseses and executes grouped tests, i.e. top-level functions contained in a package or methods contained in a class." since( "0.5.0" ) by( "Lis" ) diff --git a/source/herd/asynctest/Tester.ceylon b/source/herd/asynctest/Tester.ceylon index b212704..ed707e4 100644 --- a/source/herd/asynctest/Tester.ceylon +++ b/source/herd/asynctest/Tester.ceylon @@ -17,14 +17,18 @@ import herd.asynctest.internal { ContextThreadGroup } import herd.asynctest.runner { - AsyncTestRunner, - AsyncRunnerContext + AsyncRunnerContext, + TestInfo } import java.util.concurrent.locks { ReentrantLock } +import herd.asynctest.parameterization { + TestOutput, + TestVariantResult +} "* Performs a one test execution (test function + statements). diff --git a/source/herd/asynctest/annotations.ceylon b/source/herd/asynctest/annotations.ceylon index 9e7ad86..1542f88 100644 --- a/source/herd/asynctest/annotations.ceylon +++ b/source/herd/asynctest/annotations.ceylon @@ -22,7 +22,7 @@ import herd.asynctest.runner { } -"The same as `testExecutor(`\`class AsyncTestExecutor\``)`." +"The same as `\`testExecutor(`class AsyncTestExecutor`)`\`." see( `class AsyncTestExecutor` ) since( "0.6.0" ) by( "Lis" ) shared annotation TestExecutorAnnotation async() => testExecutor( `class AsyncTestExecutor` ); diff --git a/source/herd/asynctest/chart/package.ceylon b/source/herd/asynctest/chart/package.ceylon index b85e78b..b5aa393 100644 --- a/source/herd/asynctest/chart/package.ceylon +++ b/source/herd/asynctest/chart/package.ceylon @@ -1,5 +1,7 @@ " [[Chart]] is a set of plots, where each plot is a sequence of 2D points, [[Plot]]. + + Chart may have title, names of each axis and optional format used by report processor ([[ReportFormat]]). Plot also may have title. [[Plotter]] is used to fill in plot. diff --git a/source/herd/asynctest/evaluateAnnotatedConditions.ceylon b/source/herd/asynctest/evaluateAnnotatedConditions.ceylon index f7f4962..28f3646 100644 --- a/source/herd/asynctest/evaluateAnnotatedConditions.ceylon +++ b/source/herd/asynctest/evaluateAnnotatedConditions.ceylon @@ -28,6 +28,10 @@ import herd.asynctest.internal { findTypedAnnotations } +import herd.asynctest.parameterization { + TestOutput +} + "Evaluates test conditions applied as annotations." since( "0.0.1" ) diff --git a/source/herd/asynctest/exceptions.ceylon b/source/herd/asynctest/exceptions.ceylon index f8f85f4..87a2dcf 100644 --- a/source/herd/asynctest/exceptions.ceylon +++ b/source/herd/asynctest/exceptions.ceylon @@ -24,8 +24,10 @@ shared class FactoryReturnsNothing( String factoryTitle ) {} +"Thrown when instantiated class has neither default constructor nor factory function." since( "0.6.1" ) by( "Lis" ) shared class IncompatibleInstantiation ( + "Declaration of the trying to instantiate class." shared ClassDeclaration declaration ) extends Exception ( @@ -44,6 +46,7 @@ shared class MultipleAbortException ( extends TestAbortedException() { + "Message collected from [[abortReasons]]." shared actual String message { value message = StringBuilder(); message.append( description ); diff --git a/source/herd/asynctest/match/package.ceylon b/source/herd/asynctest/match/package.ceylon index 9af1aab..f9790e9 100644 --- a/source/herd/asynctest/match/package.ceylon +++ b/source/herd/asynctest/match/package.ceylon @@ -1,5 +1,7 @@ "Matchers are intended to organize complex test conditions into a one flexible expression. + + Each matcher is requirements specification and verification method which identifies if submitted test value satisfies matcher specification or not. diff --git a/source/herd/asynctest/parameterization/MixingEnumerator.ceylon b/source/herd/asynctest/parameterization/MixingEnumerator.ceylon index 931244e..6ebc357 100644 --- a/source/herd/asynctest/parameterization/MixingEnumerator.ceylon +++ b/source/herd/asynctest/parameterization/MixingEnumerator.ceylon @@ -1,6 +1,5 @@ -import herd.asynctest { - TestVariantResult -} + + "Enumerates mixed variants." diff --git a/source/herd/asynctest/parameterization/TestOutput.ceylon b/source/herd/asynctest/parameterization/TestOutput.ceylon new file mode 100644 index 0000000..8f5049b --- /dev/null +++ b/source/herd/asynctest/parameterization/TestOutput.ceylon @@ -0,0 +1,31 @@ +import ceylon.test { + + TestState +} + + +"Represents a one test report." +tagged( "Base" ) +see( `class TestVariantResult` ) +since( "0.0.1" ) by( "Lis" ) +shared final class TestOutput ( + "Test state of this report." shared TestState state, + "Error if occured." shared Throwable? error, + "Time in milliseconds elapsed before reporting." shared Integer elapsedTime, + "Output title." shared String title +) { + string => "``state``: ``title``"; +} + + +"Results of a one test function run with some arguments." +tagged( "Base" ) +see( `interface TestVariantEnumerator` ) +since( "0.6.0" ) by( "Lis" ) +shared final class TestVariantResult ( + "Outputs from test." shared TestOutput[] testOutput, + "Overall time in milliseconds elapsed on the test run." shared Integer overallElapsedTime, + "Overall state of the test run." shared TestState overallState +) { + string => "``overallState`` at ``overallElapsedTime``ms."; +} diff --git a/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon b/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon index 2d5341e..4480a80 100644 --- a/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon +++ b/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon @@ -1,9 +1,6 @@ import ceylon.test { TestState } -import herd.asynctest { - TestVariantResult -} "Enumerates test variants." diff --git a/source/herd/asynctest/parameterization/combinatorial.ceylon b/source/herd/asynctest/parameterization/combinatorial.ceylon index 06f99e7..a20f736 100644 --- a/source/herd/asynctest/parameterization/combinatorial.ceylon +++ b/source/herd/asynctest/parameterization/combinatorial.ceylon @@ -23,7 +23,7 @@ see( `function combinatorial`, `class ParameterizedAnnotation`, `class DataSourc `function permuting`, `function zipping`, `function mixing` ) since( "0.6.1" ) by( "Lis" ) shared final annotation class CombinatorialAnnotation ( - "Combinator function which has to return [[TestVariantEnumerator]] and takes `ArgumentVariants[]` + "Test variant generator function which has to return [[TestVariantEnumerator]] and takes `ArgumentVariants[]` which is generated based on [[DataSourceAnnotation]] at each test function argument. > Each test function argument has to be marked with [[DataSourceAnnotation]]!" shared FunctionDeclaration combinator, @@ -60,6 +60,7 @@ shared final annotation class CombinatorialAnnotation ( } + "Returns enumerator on test variants generated by [[combinator]] function." throws( `class AssertionError`, "At least one of the test function argument doesn't contain `argument` annotation or argument source function returns empty argument list." ) shared actual TestVariantEnumerator variants( FunctionDeclaration testFunction, Object? instance ) { diff --git a/source/herd/asynctest/parameterization/package.ceylon b/source/herd/asynctest/parameterization/package.ceylon index f1fcf26..d283f45 100644 --- a/source/herd/asynctest/parameterization/package.ceylon +++ b/source/herd/asynctest/parameterization/package.ceylon @@ -15,6 +15,7 @@ > Each [[TestVariant]] contains a list of generic type parameters and a list of function arguments. + ------------------------------------------- ### Custom parameterization 1. Implement [[TestVariantEnumerator]] interface: @@ -44,12 +45,14 @@ myParameterized(...) void myTest(...) {...} + ------------------------------------------- ### Varianted testing - [[parameterized]] annotation satisfies [[TestVariantProvider]] interface and + [[ParameterizedAnnotation]] satisfies [[TestVariantProvider]] interface and provides type- and value- parameterized testing based on collection of test variants. + ------------------------------------------- ### Combinatorial testing Combinatorial testing provides generation of test variants based on data sources of each test function argument. diff --git a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon index 09ab4a5..86b8299 100644 --- a/source/herd/asynctest/rule/ChainedPrePostContext.ceylon +++ b/source/herd/asynctest/rule/ChainedPrePostContext.ceylon @@ -1,6 +1,5 @@ import herd.asynctest { AsyncPrePostContext, - TestInfo, MultipleAbortException } import ceylon.collection { @@ -16,6 +15,9 @@ import java.util.concurrent.locks { import herd.asynctest.internal { CurrentThread } +import herd.asynctest.runner { + TestInfo +} "Provides in chain calls of the given prepost functions." diff --git a/source/herd/asynctest/runner/AsyncTestRunner.ceylon b/source/herd/asynctest/runner/AsyncTestRunner.ceylon index 8d81361..78959db 100644 --- a/source/herd/asynctest/runner/AsyncTestRunner.ceylon +++ b/source/herd/asynctest/runner/AsyncTestRunner.ceylon @@ -1,6 +1,4 @@ -import herd.asynctest { - TestInfo -} + "Test runner is applied to execute test function with a one test variant. diff --git a/source/herd/asynctest/runner/ChainedRunner.ceylon b/source/herd/asynctest/runner/ChainedRunner.ceylon index 7c90391..c38710b 100644 --- a/source/herd/asynctest/runner/ChainedRunner.ceylon +++ b/source/herd/asynctest/runner/ChainedRunner.ceylon @@ -1,6 +1,4 @@ -import herd.asynctest { - TestInfo -} + "Provides runners chaining: diff --git a/source/herd/asynctest/runner/CollectorContext.ceylon b/source/herd/asynctest/runner/CollectorContext.ceylon index 46bc1c7..4d82b92 100644 --- a/source/herd/asynctest/runner/CollectorContext.ceylon +++ b/source/herd/asynctest/runner/CollectorContext.ceylon @@ -1,5 +1,4 @@ -import herd.asynctest { - VariantResultBuilder, +import herd.asynctest.parameterization { TestVariantResult } diff --git a/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon b/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon index 8ef438a..2f55ff1 100644 --- a/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon +++ b/source/herd/asynctest/runner/ErrorCollectorRunner.ceylon @@ -1,6 +1,4 @@ -import herd.asynctest { - TestInfo -} + import ceylon.test.engine { MultipleFailureException } diff --git a/source/herd/asynctest/runner/RepeatRunner.ceylon b/source/herd/asynctest/runner/RepeatRunner.ceylon index ea1481c..03d729a 100644 --- a/source/herd/asynctest/runner/RepeatRunner.ceylon +++ b/source/herd/asynctest/runner/RepeatRunner.ceylon @@ -1,6 +1,7 @@ import herd.asynctest { - TestInfo, - retry, + retry +} +import herd.asynctest.parameterization { TestVariantResult } diff --git a/source/herd/asynctest/runner/RepeatStrategy.ceylon b/source/herd/asynctest/runner/RepeatStrategy.ceylon index 360053e..974514e 100644 --- a/source/herd/asynctest/runner/RepeatStrategy.ceylon +++ b/source/herd/asynctest/runner/RepeatStrategy.ceylon @@ -1,10 +1,11 @@ -import herd.asynctest { - TestVariantResult, - TestOutput -} + import ceylon.test { TestState } +import herd.asynctest.parameterization { + TestOutput, + TestVariantResult +} "Identifies if test has to be repeated or completed. diff --git a/source/herd/asynctest/TestInfo.ceylon b/source/herd/asynctest/runner/TestInfo.ceylon similarity index 96% rename from source/herd/asynctest/TestInfo.ceylon rename to source/herd/asynctest/runner/TestInfo.ceylon index f8df79a..a64951d 100644 --- a/source/herd/asynctest/TestInfo.ceylon +++ b/source/herd/asynctest/runner/TestInfo.ceylon @@ -4,12 +4,13 @@ import ceylon.language.meta.declaration { import ceylon.language.meta.model { Type } -import herd.asynctest.runner { - AsyncTestRunner -} + import herd.asynctest.internal { sequenceHash } +import herd.asynctest { + AsyncPrePostContext +} "Information on currently running test variant." diff --git a/source/herd/asynctest/VariantResultBuilder.ceylon b/source/herd/asynctest/runner/VariantResultBuilder.ceylon similarity index 94% rename from source/herd/asynctest/VariantResultBuilder.ceylon rename to source/herd/asynctest/runner/VariantResultBuilder.ceylon index 2b81ff5..6a2b31d 100644 --- a/source/herd/asynctest/VariantResultBuilder.ceylon +++ b/source/herd/asynctest/runner/VariantResultBuilder.ceylon @@ -8,12 +8,16 @@ import ceylon.test.engine { TestSkippedException, TestAbortedException } +import herd.asynctest.parameterization { + TestOutput, + TestVariantResult +} "Builds test variant result from several test outputs." see( `class TestVariantResult`, `class TestOutput` ) since( "0.6.0" ) by( "Lis" ) -shared class VariantResultBuilder() { +final class VariantResultBuilder() { variable Integer startTime = system.nanoseconds; variable TestState overallState = TestState.skipped; ArrayList outs = ArrayList(); diff --git a/source/herd/asynctest/variantEnumerators.ceylon b/source/herd/asynctest/variantEnumerators.ceylon index e2317a5..7a2a7d4 100644 --- a/source/herd/asynctest/variantEnumerators.ceylon +++ b/source/herd/asynctest/variantEnumerators.ceylon @@ -1,13 +1,14 @@ import herd.asynctest.parameterization { TestVariant, - TestVariantEnumerator + TestVariantEnumerator, + TestVariantResult } "Test variant without any arguments." since( "0.6.0" ) by( "Lis" ) -shared object emptyTestVariant extends TestVariant( [], [] ) { +object emptyTestVariant extends TestVariant( [], [] ) { shared actual String variantName = ""; shared actual Boolean equals( Object that ) { if ( is TestVariant that ) { @@ -24,7 +25,7 @@ shared object emptyTestVariant extends TestVariant( [], [] ) { "Provides a one empty test variant." since( "0.6.0" ) by( "Lis" ) -shared class EmptyTestVariantEnumerator() satisfies TestVariantEnumerator { +class EmptyTestVariantEnumerator() satisfies TestVariantEnumerator { variable TestVariant|Finished currentVal = emptyTestVariant; shared actual TestVariant|Finished current => currentVal; shared actual void moveNext(TestVariantResult result) => currentVal = finished; @@ -33,7 +34,7 @@ shared class EmptyTestVariantEnumerator() satisfies TestVariantEnumerator { "Combines several test variant iterators." since( "0.6.0" ) by( "Lis" ) -shared class CombinedVariantEnumerator( Iterator providers ) +class CombinedVariantEnumerator( Iterator providers ) satisfies TestVariantEnumerator { From 49148497bfb362ce9c6129948d9c3b5b69151260 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 11 Jan 2017 18:37:11 +0300 Subject: [PATCH 51/72] repackaging + benchmark --- README.md | 7 +- .../benchmark/isValueOrFunction.ceylon | 64 +++++++ .../benchmark/isWrapperOrFunction.ceylon | 146 +++++++++++++++ .../asynctest/benchmark/package.ceylon | 5 + .../asynctest/benchmark/plusMinus.ceylon | 39 ++++ .../herd/examples/asynctest/module.ceylon | 5 +- .../herd/asynctest/benchmark/BaseBench.ceylon | 166 ++++++++++++++++++ source/herd/asynctest/benchmark/Bench.ceylon | 21 +++ source/herd/asynctest/benchmark/Clock.ceylon | 105 +++++++++++ .../benchmark/ComparativeStatistic.ceylon | 71 ++++++++ .../benchmark/CompletionCriterion.ceylon | 83 +++++++++ .../benchmark/EmptyParameterBench.ceylon | 25 +++ .../asynctest/benchmark/ErrorCriteria.ceylon | 79 +++++++++ .../asynctest/benchmark/GCStrategy.ceylon | 63 +++++++ .../benchmark/IntegerComparator.ceylon | 19 ++ .../benchmark/IterationsCriteria.ceylon | 59 +++++++ .../herd/asynctest/benchmark/Options.ceylon | 21 +++ .../benchmark/ParameterResult.ceylon | 115 ++++++++++++ source/herd/asynctest/benchmark/Result.ceylon | 118 +++++++++++++ .../asynctest/benchmark/SingleBench.ceylon | 25 +++ source/herd/asynctest/benchmark/Stage.ceylon | 53 ++++++ .../benchmark/StatisticSummary.ceylon | 106 +++++++++++ .../benchmark/ThreadLocalClock.ceylon | 111 ++++++++++++ .../asynctest/benchmark/TimeCriteria.ceylon | 80 +++++++++ .../herd/asynctest/benchmark/TimeUnit.ceylon | 61 +++++++ .../herd/asynctest/benchmark/benchmark.ceylon | 74 ++++++++ .../herd/asynctest/benchmark/blackHole.ceylon | 49 ++++++ .../asynctest/benchmark/contextWriters.ceylon | 120 +++++++++++++ .../asynctest/benchmark/delegateWriter.ceylon | 21 +++ .../herd/asynctest/benchmark/package.ceylon | 121 +++++++++++++ .../stringifyNumberOfOperations.ceylon | 12 ++ source/herd/asynctest/exceptions.ceylon | 4 +- .../internal/MarkerThreadGroup.ceylon | 2 +- .../herd/asynctest/internal/SimpleStat.ceylon | 65 +++++++ .../StatisticCalculator.ceylon} | 45 ++--- .../internal/declarationVerifier.ceylon | 2 +- .../herd/asynctest/match/checkMatchers.ceylon | 8 +- .../asynctest/match/iterableMatchers.ceylon | 2 +- .../herd/asynctest/match/listMatchers.ceylon | 12 +- .../herd/asynctest/match/mapMatchers.ceylon | 6 +- .../asynctest/match/numberMatchers.ceylon | 8 +- .../herd/asynctest/match/setMatchers.ceylon | 8 +- source/herd/asynctest/module.ceylon | 18 +- .../parameterization/ArgumentVariants.ceylon | 2 +- .../parameterization/CombinatorialKind.ceylon | 10 +- .../parameterization/MixingEnumerator.ceylon | 2 +- .../TestVariantEnumerator.ceylon | 2 +- .../parameterization/combinatorial.ceylon | 4 +- .../parameterization/dataSource.ceylon | 8 +- .../mixingCombinations.ceylon | 6 +- .../asynctest/parameterization/package.ceylon | 24 ++- .../permutingCombinations.ceylon | 6 +- .../zippingCombinations.ceylon | 6 +- .../asynctest/rule/ChainedTestContext.ceylon | 2 +- .../herd/asynctest/rule/ContextualRule.ceylon | 2 +- .../asynctest/rule/CurrentTestStore.ceylon | 2 +- source/herd/asynctest/rule/MeterRule.ceylon | 8 +- source/herd/asynctest/rule/RuleChain.ceylon | 2 +- source/herd/asynctest/rule/SignalRule.ceylon | 2 +- .../herd/asynctest/rule/StatisticRule.ceylon | 8 +- source/herd/asynctest/rule/exceptions.ceylon | 2 +- .../herd/asynctest/rule/ruleAnnotation.ceylon | 4 +- .../runner/AsyncRunnerContext.ceylon | 2 +- 63 files changed, 2223 insertions(+), 105 deletions(-) create mode 100644 examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon create mode 100644 examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon create mode 100644 examples/herd/examples/asynctest/benchmark/package.ceylon create mode 100644 examples/herd/examples/asynctest/benchmark/plusMinus.ceylon create mode 100644 source/herd/asynctest/benchmark/BaseBench.ceylon create mode 100644 source/herd/asynctest/benchmark/Bench.ceylon create mode 100644 source/herd/asynctest/benchmark/Clock.ceylon create mode 100644 source/herd/asynctest/benchmark/ComparativeStatistic.ceylon create mode 100644 source/herd/asynctest/benchmark/CompletionCriterion.ceylon create mode 100644 source/herd/asynctest/benchmark/EmptyParameterBench.ceylon create mode 100644 source/herd/asynctest/benchmark/ErrorCriteria.ceylon create mode 100644 source/herd/asynctest/benchmark/GCStrategy.ceylon create mode 100644 source/herd/asynctest/benchmark/IntegerComparator.ceylon create mode 100644 source/herd/asynctest/benchmark/IterationsCriteria.ceylon create mode 100644 source/herd/asynctest/benchmark/Options.ceylon create mode 100644 source/herd/asynctest/benchmark/ParameterResult.ceylon create mode 100644 source/herd/asynctest/benchmark/Result.ceylon create mode 100644 source/herd/asynctest/benchmark/SingleBench.ceylon create mode 100644 source/herd/asynctest/benchmark/Stage.ceylon create mode 100644 source/herd/asynctest/benchmark/StatisticSummary.ceylon create mode 100644 source/herd/asynctest/benchmark/ThreadLocalClock.ceylon create mode 100644 source/herd/asynctest/benchmark/TimeCriteria.ceylon create mode 100644 source/herd/asynctest/benchmark/TimeUnit.ceylon create mode 100644 source/herd/asynctest/benchmark/benchmark.ceylon create mode 100644 source/herd/asynctest/benchmark/blackHole.ceylon create mode 100644 source/herd/asynctest/benchmark/contextWriters.ceylon create mode 100644 source/herd/asynctest/benchmark/delegateWriter.ceylon create mode 100644 source/herd/asynctest/benchmark/package.ceylon create mode 100644 source/herd/asynctest/benchmark/stringifyNumberOfOperations.ceylon create mode 100644 source/herd/asynctest/internal/SimpleStat.ceylon rename source/herd/asynctest/{rule/Stat.ceylon => internal/StatisticCalculator.ceylon} (81%) diff --git a/README.md b/README.md index 370b591..19569de 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ is an extension to SDK `ceylon.test` module with following capabilities: * reporting test results using charts (or plots) The module is available on [Ceylon Herd](https://herd.ceylon-lang.org/modules/herd.asynctest). -Current version is 0.6.0. +Current version is 0.7.0. #### Ceylon compiler / platform @@ -34,7 +34,7 @@ Available on JVM only #### Usage and documentation The extension is aimed to be run using Ceylon test tool. -See usage details in [API documentation](https://modules.ceylon-lang.org/repo/1/herd/asynctest/0.6.0/module-doc/api/index.html). +See usage details in [API documentation](https://modules.ceylon-lang.org/repo/1/herd/asynctest/0.7.0/module-doc/api/index.html). #### Examples @@ -42,8 +42,9 @@ See usage details in [API documentation](https://modules.ceylon-lang.org/repo/1/ * Test of [Fibonacci numbers calculation](examples/herd/examples/asynctest/fibonacci). Calculation function is executed on separated thread and returns results using `ceylon.promise`. * [Time scheduler](examples/herd/examples/asynctest/scheduler) testing. -* [Microbenchmark](examples/herd/examples/asynctest/mapperformance) - +* [Comparative performance](examples/herd/examples/asynctest/mapperformance) - comparative performance test of Ceylon / Java HashMap and TreeMap. +* [Benchmarking](examples/herd/examples/asynctest/benchmark) examples of benchmarks. * [Matchers](examples/herd/examples/asynctest/matchers) - matchers usage. * [Parameterized](examples/herd/examples/asynctest/parameterized) - type- and value- parameterized testing. * [Rules](examples/herd/examples/asynctest/rule) - test rules usage. diff --git a/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon new file mode 100644 index 0000000..fb2ccfa --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon @@ -0,0 +1,64 @@ +import herd.asynctest { + AsyncTestContext, + async +} +import ceylon.test { + test +} +import herd.asynctest.benchmark { + writeRelativeToFastest, + benchmark, + SingleBench, + Options, + TotalIterations +} + + +// benchmark parameters +Integer integerValue => 1; +Integer integerFunction() => 1; + + +"Benchmark function - narrow to `Integer` firstly." +Boolean isValue(Integer|Integer() source) { + if (is Integer source) {return true;} + else {return false;} +} + +"Benchmark function - narrow to `Integer()` firstly." +Boolean isFunction(Integer|Integer() source) { + if (is Integer() source) {return true;} + else {return false;} +} + + +"What is faster narrowing to function of to class?: + + Integer|Integer() source => ... + + ... + + if (is Integer source) {return true; } + else {return false;} + + or + + if (is Integer() source) {return true; } + else {return false;} + + " +shared test async void isValueOrFunction(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options( TotalIterations( 10000 ), TotalIterations( 100 ) ), + [ + SingleBench("value", isValue), + SingleBench("function", isFunction) + ], + [integerValue], [integerFunction] + ) + ); + context.complete(); +} + diff --git a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon new file mode 100644 index 0000000..2e6c006 --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon @@ -0,0 +1,146 @@ +import herd.asynctest { + AsyncTestContext, + async +} +import ceylon.test { + test +} +import herd.asynctest.benchmark { + benchmark, + writeRelativeToFastest, + EmptyParameterBench, + TimeUnit, + Options, + Clock, + TotalIterations, + LocalIterations, + LocalError +} + + +"Wrapper to test narrow SingleArgumentWrapper to SingleArgumentWrapper. + Supposes the function has just a one argument." +class SingleArgumentWrapper ( + shared Anything(T) func +){} + +"Wrapper to test narrow MultipleArgumentWrapper to MultipleArgumentWrapper<[Something...]>. + The function may have multiple arguments." +class MultipleArgumentWrapper ( + shared Anything(*T) func +) + given T satisfies Anything[] +{} + +"Test function to narrow Anything(Nothing) to." +void integerArgumentFunction(Integer i) {} + +"Test function to narrow Anything(Nothing) to." +void floatArgumentFunction(Float i) {} + + +"Narrows `SingleArgumentWrapper` to `SingleArgumentWrapper`." +Boolean narrowSingleWrapperToInteger(SingleArgumentWrapper i)() { + if (is SingleArgumentWrapper i) {return true;} + else {return false;} +} + +"Narrows `MultipleArgumentWrapper` to `MultipleArgumentWrapper<[Integer]>`." +Boolean narrowMultipleWrapperToInteger(MultipleArgumentWrapper i)() { + if (is MultipleArgumentWrapper<[Integer]> i) {return true;} + else {return false;} +} + +"Narrows `Anything(Nothing)` to `Anything(Integer)`." +Boolean narrowFunctionToInteger(Anything(Nothing) f)() { + if (is Anything(Integer) f) {return true;} + else {return false;} +} + + +"Executes benchmark tests for: + * [[narrowSingleWrapperToInteger]] + * [[narrowMultipleWrapperToInteger]] + * [[narrowFunctionToInteger]] + + Uses CPU clock. + " +shared test async void isWrapperOrFunctionCPU(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options( LocalIterations( 10000 ).or( LocalError( 0.005 ) ), LocalIterations( 1000 ), TimeUnit.milliseconds, Clock.cpu ), + [ + EmptyParameterBench ( + "narrowed single argument wrapper", + narrowSingleWrapperToInteger(SingleArgumentWrapper(integerArgumentFunction)) + ), + EmptyParameterBench ( + "not narrowed single argument wrapper", + narrowSingleWrapperToInteger(SingleArgumentWrapper(floatArgumentFunction)) + ), + EmptyParameterBench ( + "narrowed multiple argument wrapper", + narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Integer]>(integerArgumentFunction)) + ), + EmptyParameterBench ( + "not narrowed multiple argument wrapper", + narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Float]>(floatArgumentFunction)) + ), + EmptyParameterBench ( + "narrowed function", + narrowFunctionToInteger(integerArgumentFunction) + ), + EmptyParameterBench ( + "not narrowed function", + narrowFunctionToInteger(floatArgumentFunction) + ) + ] + ) + ); + context.complete(); +} + + +"Executes benchmark tests for: + * [[narrowSingleWrapperToInteger]] + * [[narrowMultipleWrapperToInteger]] + * [[narrowFunctionToInteger]] + + Uses wall clock. + " +shared test async void isWrapperOrFunctionWall(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options( TotalIterations( 10000 ), TotalIterations( 1000 ), TimeUnit.milliseconds, Clock.wall ), + [ + EmptyParameterBench ( + "narrowed single argument wrapper", + narrowSingleWrapperToInteger(SingleArgumentWrapper(integerArgumentFunction)) + ), + EmptyParameterBench ( + "not narrowed single argument wrapper", + narrowSingleWrapperToInteger(SingleArgumentWrapper(floatArgumentFunction)) + ), + EmptyParameterBench ( + "narrowed multiple argument wrapper", + narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Integer]>(integerArgumentFunction)) + ), + EmptyParameterBench ( + "not narrowed multiple argument wrapper", + narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Float]>(floatArgumentFunction)) + ), + EmptyParameterBench ( + "narrowed function", + narrowFunctionToInteger(integerArgumentFunction) + ), + EmptyParameterBench ( + "not narrowed function", + narrowFunctionToInteger(floatArgumentFunction) + ) + ] + ) + ); + context.complete(); +} diff --git a/examples/herd/examples/asynctest/benchmark/package.ceylon b/examples/herd/examples/asynctest/benchmark/package.ceylon new file mode 100644 index 0000000..8f16693 --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/package.ceylon @@ -0,0 +1,5 @@ + +"Benchmarking examples + " +by( "Lis" ) +shared package herd.examples.asynctest.benchmark; diff --git a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon new file mode 100644 index 0000000..93491f2 --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon @@ -0,0 +1,39 @@ +import ceylon.test { + + test +} +import herd.asynctest { + + async, + AsyncTestContext +} +import herd.asynctest.benchmark { + + writeRelativeToFastest, + benchmark, + LocalIterations, + Options, + SingleBench, + LocalError +} + + +Integer plusBenchmarkFunction(Integer x, Integer y) { + return x + y; +} +Integer minusBenchmarkFunction(Integer x, Integer y) { + return x - y; +} + +shared test async void plusMinusBenchmark(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options(LocalIterations(20000).or(LocalError(0.002))), + [SingleBench("plus", plusBenchmarkFunction), + SingleBench("minus", minusBenchmarkFunction)], + [1, 1], [2, 3], [25, 34] + ) + ); + context.complete(); +} diff --git a/examples/herd/examples/asynctest/module.ceylon b/examples/herd/examples/asynctest/module.ceylon index 3be0d6a..d527cfd 100644 --- a/examples/herd/examples/asynctest/module.ceylon +++ b/examples/herd/examples/asynctest/module.ceylon @@ -1,5 +1,6 @@ "Contains some examples of `asyncTest` usage: + * [[package herd.examples.asynctest.benchmark]] - examples of benchmark test * [[package herd.examples.asynctest.fibonacci]] - test of calculation of Fibonacci numbers in separated thread * [[package herd.examples.asynctest.mapperformance]] - comparative performance test of Ceylon - Java HashMap and TreeMap * [[package herd.examples.asynctest.matchers]] - several examples of matchers from `package herd.asynctest.match` @@ -31,7 +32,7 @@ license ( ) by( "Lis" ) native( "jvm" ) -module herd.examples.asynctest "0.6.1" { +module herd.examples.asynctest "0.7.0" { shared import ceylon.collection "1.3.1"; shared import ceylon.http.server "1.3.1"; shared import ceylon.http.client "1.3.1"; @@ -41,6 +42,6 @@ module herd.examples.asynctest "0.6.1" { shared import ceylon.file "1.3.1"; shared import ceylon.promise "1.3.1"; shared import ceylon.test "1.3.1"; - shared import herd.asynctest "0.6.1"; + shared import herd.asynctest "0.7.0"; shared import java.base "8"; } diff --git a/source/herd/asynctest/benchmark/BaseBench.ceylon b/source/herd/asynctest/benchmark/BaseBench.ceylon new file mode 100644 index 0000000..99139c1 --- /dev/null +++ b/source/herd/asynctest/benchmark/BaseBench.ceylon @@ -0,0 +1,166 @@ +import java.lang.management { + ManagementFactory +} +import herd.asynctest.internal { + SimpleStat +} + + +"Intended to implement details of bench execution in a signle thread. + + Just [[runIteration]] has to be implemented which is indended to invoke test function once. + See [[execute]] for the run cycle. + " +tagged( "Bench" ) +since( "0.7.0" ) by( "Lis" ) +shared abstract class BaseBench ( + shared actual String title +) + satisfies Bench + given Parameter satisfies Anything[] +{ + + variable Integer? memoizedHash = null; + + shared actual default Integer hash => memoizedHash else ( memoizedHash = 7 * title.hash + 37 ); + + + "Executed _before warmup round_ started. By default runs garbage collector." + shared default void beforeWarmupRound( Options options ) { + options.gcStrategy.gc( Stage.beforeWarmupRound ); + } + + "Executed _after warmup round_ has been completed. By default do nothing." + shared default void afterWarmupRound( Options options ) { + options.gcStrategy.gc( Stage.afterWarmupRound ); + } + + "Executed _before each_ warmup iteration. By default do nothing." + shared default void beforeWarmupIteration( Options options ) { + options.gcStrategy.gc( Stage.beforeWarmupIteration ); + } + + "Executed _after each_ warmup iteration. By default do nothing." + shared default void afterWarmupIteration( Options options ) { + options.gcStrategy.gc( Stage.afterWarmupIteration ); + } + + "Executed _before all_ measurements started. By default runs garbage collector." + shared default void beforeMeasureRound( Options options ) { + options.gcStrategy.gc( Stage.beforeMeasureRound ); + } + + "Executed _after all_ measurements have been completed. By default do nothing." + shared default void afterMeasureRound( Options options ) { + options.gcStrategy.gc( Stage.afterMeasureRound ); + } + + "Executed _before each_ iteration. By default do nothing." + shared default void beforeMeasureIteration( Options options ) { + options.gcStrategy.gc( Stage.beforeMeasureIteration ); + } + + "Executed _after each_ iteration. By default do nothing." + shared default void afterMeasureIteration( Options options ) { + options.gcStrategy.gc( Stage.afterMeasureIteration ); + } + + "Invokes a one test iteration. + May return results in order to avoid dead-code elimination. + Returned result is consumed using [[pushToBlackHole]]." + shared formal Anything(*Parameter) runIteration; + + + "Returns number of GC runs up to now." + Integer numberOfGCRuns() { + value gcBeans = ManagementFactory.garbageCollectorMXBeans; + variable Integer numGC = 0; + for ( gcBean in gcBeans ) { numGC += gcBean.collectionCount; } + return numGC; + } + + "Executes test using the following cycle: + * [[beforeWarmupRound]] + * Cycle while [[Options.warmupCriterion]] is not met: + * [[beforeWarmupIteration]] + * [[runIteration]] + * consume result returned by [[runIteration]] using [[pushToBlackHole]] + * [[afterWarmupIteration]] + * [[afterWarmupRound]] + * [[beforeMeasureRound]] + * Cycle while [[Options.measureCriterion]] is not met: + * [[beforeMeasureIteration]] + * [[runIteration]] + * collect time statistic + * consume result returned by [[runIteration]] using [[pushToBlackHole]] + * [[afterMeasureIteration]] + * [[afterMeasureRound]] + + > Note: runs when GC wroks are excluded from statistic calculations. + " + shared actual StatisticSummary execute ( + Options options, Parameter parameter + ) { + + // intialize clock + Clock clock = options.clock; + clock.initialize(); + // factor to scale time delta from nanoseconds (measured in) to timeUnit + Float timeFactor = TimeUnit.nanoseconds.factorToSeconds / options.timeUnit.factorToSeconds; + + // warmup round - clock is also warmupped! + if ( exists warmupCriterion = options.warmupCriterion ) { + SimpleStat calculator = SimpleStat(); + beforeWarmupRound( options ); + while ( true ) { + beforeWarmupIteration( options ); + clock.start(); + Anything ret = runIteration( *parameter ); + Float delta = clock.measure() * timeFactor; + if ( delta > 0.0 ) { + calculator.sample( 1.0 / delta ); + if ( warmupCriterion.verify( delta, calculator.result, options.timeUnit ) ) { + afterWarmupIteration( options ); + break; + } + } + afterWarmupIteration( options ); + pushToBlackHole( ret ); + } + afterWarmupRound( options ); + } + + // measure iterations + SimpleStat calculator = SimpleStat(); + beforeMeasureRound( options ); + while ( true ) { + // number of GC starts before test run + Integer numGCBefore = numberOfGCRuns(); + // execute the test + beforeMeasureIteration( options ); + clock.start(); + Anything ret = runIteration( *parameter ); + Float delta = clock.measure() * timeFactor; + // calculate execution statistic + if ( delta > 0.0 ) { + // number of GC starts after test run + Integer numGCAfter = numberOfGCRuns(); + if ( numGCAfter == numGCBefore ) { + // add sample only if GC has not been started during the test + calculator.sample( 1.0 / delta ); + if ( options.measureCriterion.verify( delta, calculator.result, options.timeUnit ) ) { + // round is completed + afterMeasureIteration( options ); + break; + } + } + } + // completing iteration + afterMeasureIteration( options ); + pushToBlackHole( ret ); + } + afterMeasureRound( options ); + return calculator.result; + } + +} diff --git a/source/herd/asynctest/benchmark/Bench.ceylon b/source/herd/asynctest/benchmark/Bench.ceylon new file mode 100644 index 0000000..5cd2c30 --- /dev/null +++ b/source/herd/asynctest/benchmark/Bench.ceylon @@ -0,0 +1,21 @@ + + +"Represents benchmark test parameterized with `Parameter` type." +tagged( "Bench" ) +see( `function benchmark` ) +since( "0.7.0" ) by( "Lis" ) +shared interface Bench + given Parameter satisfies Anything[] +{ + + "Bench title. Generally unique." + shared formal String title; + + "Executes this bench with the given `Parameter` value. + Returns statistic of operations per second for the execution." + shared formal StatisticSummary execute ( + "Options of the bench execution." Options options, + "Value of the parameter the bench is parameterized with." Parameter parameter + ); + +} diff --git a/source/herd/asynctest/benchmark/Clock.ceylon b/source/herd/asynctest/benchmark/Clock.ceylon new file mode 100644 index 0000000..2b30440 --- /dev/null +++ b/source/herd/asynctest/benchmark/Clock.ceylon @@ -0,0 +1,105 @@ +import java.lang.management { + ManagementFactory +} +import java.lang { + ThreadLocal +} +import java.util.\ifunction { + Supplier +} + + +"Measures the time interval with one of the three strategies: + + 1. [[wall]] time, i.e. the time provided by system wall clocks. + 2. [[cpu]] thread time which is time the thread occupies a CPU. + 3. Thread [[user]] time which is time the thread occupies a CPU in user mode, rather than in system mode. + + > Note: `cpu` and `user` times are evaluated approximately. + + + Usage: + 1. Call [[initialize]] before measure cycle. + 2. Call [[start]] before each measure. + 3. Call [[measure]] to measure time interval from the last [[start]] call. + + Example: + + Clock clock = Clock.wall; + clock.initialize(); + for ( item in 0 : totalRuns ) { + clock.start(); + doOperation(); + Integer interval = clock.measure(); + } + + Time interval is measured relative to the thread current to the caller. + So, if [[measure]] is called from _thread A_, then the interval is measured from the last call of [[start]] + done on the same _thread A_. + " +tagged( "Options" ) +see( `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared class Clock + of cpu | user | wall +{ + + "Instantiates new local to the thread clock." + ThreadLocalClock() instantiate; + + shared actual String string; + + + "Indicates that `ThreadMXBean.currentThreadCpuTime` has to be used for time measurement. + If CPU time is not supported uses `system.nanoseconds`." + shared new cpu { + // TODO: log clock selection? + if ( ManagementFactory.threadMXBean.threadCpuTimeSupported ) { + string = "CPU clock"; + instantiate = CPULocalClock; + } + else { + string = "wall clock"; + instantiate = WallLocalClock; + } + } + + "Indicates that `ThreadMXBean.currentThreadUserTime` has to be used for time measurement. + If CPU time is not supported uses `system.nanoseconds`." + shared new user { + // TODO: log clock selection? + if ( ManagementFactory.threadMXBean.threadCpuTimeSupported ) { + string = "CPU user clock"; + instantiate = UserLocalClock; + } + else { + string = "wall clock"; + instantiate = WallLocalClock; + } + } + + "Indicates that `system.nanoseconds` has to be used for time measurement." + shared new wall { + string = "wall clock"; + instantiate = WallLocalClock; + } + + + ThreadLocal localClock = ThreadLocal.withInitial ( + object satisfies Supplier { + get() => instantiate(); + } + ); + + + "Initializes the clock locally for the current thread. + Has to be called before any measurements." + shared void initialize() => localClock.get().initialize(); + + "Starts time interval measure from the current time and locally for the current thread." + shared void start() => localClock.get().start(); + + "Measures time interval from the last [[start]] call (in the same thread as `measure` called) and up to now." + shared Integer measure() => localClock.get().measure(); + +} diff --git a/source/herd/asynctest/benchmark/ComparativeStatistic.ceylon b/source/herd/asynctest/benchmark/ComparativeStatistic.ceylon new file mode 100644 index 0000000..2cfa627 --- /dev/null +++ b/source/herd/asynctest/benchmark/ComparativeStatistic.ceylon @@ -0,0 +1,71 @@ +import java.lang { + Math +} + + +"Represents statistic values comparing to another [[StatisticSummary]]." +tagged( "Result" ) +see( `class StatisticSummary` ) +since( "0.7.0" ) by( "Lis" ) +shared final class ComparativeStatistic ( + "Statistic to be compared to [[baseMean]]." shared StatisticSummary stat, + "Base mean value the [[stat]] is compared to." shared Float baseMean +) { + + "Minimum of the values that have been statisticaly treated." + shared Float min => stat.min; + + "Maximum of the values that have been statisticaly treated." + shared Float max => stat.max; + + "Mean value." + shared Float mean => stat.mean; + + "Returns standard deviation of the values that have been statisticaly treated. + Standard deviation is `variance^0.5`." + shared Float standardDeviation => stat.standardDeviation; + + "The number of the values that have been statisticaly treated." + shared Integer size => stat.size; + + "Variance of the values that have been statisticaly treated. + The variance is mean((x-mean(x))^2)." + shared Float variance => stat.variance; + + "Sample variance is size/(size - 1)*variance." + shared Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; + + "Sample standard deviation is sqrt(size/(size - 1)*variance)." + shared Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) ) * standardDeviation else 0.0; + + "Equals to standardDeviation/sqrt(size)." + shared Float standardError => if ( size > 1 ) then standardDeviation / Math.sqrt( size.float ) else 0.0; + + "Equals to sampleStandardDeviation/sqrt(size)." + shared Float sampleError => if ( size > 1 ) then standardDeviation / Math.sqrt( ( size - 1 ).float ) else 0.0; + + "Equals to sampleError/mean." + shared Float relativeSampleError => sampleError / mean; + + + "Mean value relative to `baseMean` in percents." + shared Integer relativeMean => if ( mean == baseMean ) then 100 else ( mean / baseMean * 100 ).integer; + + + shared actual Boolean equals( Object other ) { + if ( is ComparativeStatistic other ) { + return stat == other.stat && baseMean == other.baseMean; + } + else { + return false; + } + } + + shared actual Integer hash => stat.hash * 37 + baseMean.integer; + + + shared actual String string => "mean=``Float.format(mean, 0, 3)``, standard deviation=``Float.format(standardDeviation, 0, 3)``, " + + "max=``Float.format(max, 0, 3)``, min=``Float.format(min, 0, 3)``, total samples=``size``,"+ + "relative mean=``relativeMean``%, base mean=``Float.format(baseMean, 0, 3)``"; + +} diff --git a/source/herd/asynctest/benchmark/CompletionCriterion.ceylon b/source/herd/asynctest/benchmark/CompletionCriterion.ceylon new file mode 100644 index 0000000..1c0af8f --- /dev/null +++ b/source/herd/asynctest/benchmark/CompletionCriterion.ceylon @@ -0,0 +1,83 @@ + + +"Identifies if benchmark iterations have to be completed or continued." +tagged( "Options", "Criteria" ) +see( `function benchmark`, `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared interface CompletionCriterion { + + "Resets criterion to an initial state." + shared formal void reset(); + + "Returns `true` if completion criterion is met and `false` otherwise." + shared formal Boolean verify ( + "The latest execution time in the given time unit." Float delta, + "Accumulated execution statistic of operations per given time unit." StatisticSummary result, + "Time unit [[delta]] and [[result]] are measured in." TimeUnit timeUnit + ); + + "Combines `this` and `other` criterion using logical `and`. + I.e. returned criterion completes only if both `this` and `other` are completed." + shared default CompletionCriterion and( CompletionCriterion other ) { + return AndCriterion( this, other ); + } + + "Combines `this` and `other` criterion using logical `or`. + I.e. returned criterion completes if either `this` or `other` is completed." + shared default CompletionCriterion or( CompletionCriterion other ) { + return OrCriterion( this, other ); + } + +} + + +"Logical and of two criteria." +since( "0.7.0" ) by( "Lis" ) +class AndCriterion( CompletionCriterion+ criteria ) satisfies CompletionCriterion { + + shared actual void reset() { + for ( item in criteria ) { + item.reset(); + } + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + for ( item in criteria ) { + if ( !item.verify( delta, result, timeUnit ) ) { + return false; + } + } + return true; + } + + shared actual CompletionCriterion and( CompletionCriterion other ) { + return AndCriterion( other, *criteria ); + } + +} + + +"Logical or of two criteria." +since( "0.7.0" ) by( "Lis" ) +class OrCriterion( CompletionCriterion+ criteria ) satisfies CompletionCriterion { + + shared actual void reset() { + for ( item in criteria ) { + item.reset(); + } + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + for ( item in criteria ) { + if ( item.verify( delta, result, timeUnit ) ) { + return true; + } + } + return false; + } + + shared actual CompletionCriterion or( CompletionCriterion other ) { + return OrCriterion( other, *criteria ); + } + +} diff --git a/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon b/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon new file mode 100644 index 0000000..8f93c40 --- /dev/null +++ b/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon @@ -0,0 +1,25 @@ + +"Executes test function in a single thread. + Test function takes no argument - the same as to `SingleBench<[]>`." +tagged( "Bench" ) +see( `function benchmark`, `class SingleBench` ) +since( "0.7.0" ) by( "Lis" ) +shared final class EmptyParameterBench ( + "Bench title. Generally unique." + String title, + "Function to be tested." + shared actual Anything() runIteration +) + extends BaseBench<[]>( title ) +{ + + shared actual Boolean equals( Object other ) { + if ( is EmptyParameterBench other ) { + return title == other.title; + } + else { + return false; + } + } + +} diff --git a/source/herd/asynctest/benchmark/ErrorCriteria.ceylon b/source/herd/asynctest/benchmark/ErrorCriteria.ceylon new file mode 100644 index 0000000..c2f331c --- /dev/null +++ b/source/herd/asynctest/benchmark/ErrorCriteria.ceylon @@ -0,0 +1,79 @@ +import java.util.concurrent { + ConcurrentSkipListMap +} +import java.lang { + Thread, + Math +} + + +"Continues benchmark iterations while total (over all threads) relative sample error doesn't exceed [[maxRelativeError]]. + I.e. [[StatisticSummary.relativeSampleError]] is compared against [[maxRelativeError]]." +tagged( "Criteria" ) +see( `class Options`, `class LocalError` ) +throws( `class AssertionError`, "Maximum allowed total error is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class TotalError ( + "Maximum allowed total error. Has to be > 0." Float maxRelativeError, + "Minimum number of iterations before check error. Default is 10." Integer minIterations = 10 +) + satisfies CompletionCriterion +{ + + "Maximum allowed total error has to be > 0." + assert( maxRelativeError > 0.0 ); + + ConcurrentSkipListMap localStat = ConcurrentSkipListMap( IntegerComparator() ); + + + shared actual void reset() { + localStat.clear(); + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + Integer id = Thread.currentThread().id; + localStat.put( id, result ); + variable Integer size = 0; + variable Float mean = 0.0; + variable Float variance = 0.0; + for ( item in localStat.values() ) { + size += item.size; + mean += item.mean; + variance += item.variance; + } + if ( size > minIterations ) { + return Math.sqrt( variance / ( size - 1 ) ) / mean < maxRelativeError; + } + else { + return false; + } + } + +} + + +"Continues benchmark iterations while thread local relative sample error doesn't exceed [[maxRelativeError]]. + I.e. [[StatisticSummary.relativeSampleError]] is compared against [[maxRelativeError]]." +tagged( "Criteria" ) +see( `class Options`, `class TotalError` ) +throws( `class AssertionError`, "Maximum allowed local error is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class LocalError ( + "Maximum allowed local error. Has to be > 0." Float maxRelativeError, + "Minimum number of iterations before check error. Default is 10." Integer minIterations = 10 +) + satisfies CompletionCriterion +{ + + "Maximum allowed local error has to be > 0." + assert( maxRelativeError > 0.0 ); + + + shared actual void reset() { + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + return result.size > minIterations && result.relativeSampleError < maxRelativeError; + } + +} diff --git a/source/herd/asynctest/benchmark/GCStrategy.ceylon b/source/herd/asynctest/benchmark/GCStrategy.ceylon new file mode 100644 index 0000000..7287e2f --- /dev/null +++ b/source/herd/asynctest/benchmark/GCStrategy.ceylon @@ -0,0 +1,63 @@ +import java.lang { + System +} + + +"Strategy to run garbage collector." +tagged( "Options" ) +see( `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared interface GCStrategy { + "Runs garbage collector if all conditions meet the strategy." + shared formal void gc( "Current execution stage." Stage stage ); +} + + +"Identifies at which execution stages garbage collector has to be run." +tagged( "Options" ) +see( `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared final class GCStagedStrategy satisfies GCStrategy { + + Stage[] stageSet; + + + "Runs GC at a number of stages." + shared new combined( Stage+ stages ) { + stageSet = stages; + } + + "GC is to be never run." + shared new never { + stageSet = empty; + } + + "Runs GC before warmup round only." + shared new beforeWarmupRound { + stageSet = [Stage.beforeWarmupRound]; + } + + "Runs GC before measure round only." + shared new beforeMeasureRound { + stageSet = [Stage.beforeMeasureRound]; + } + + "Runs GC before warmup and measure rounds." + shared new beforeRounds { + stageSet = [Stage.beforeWarmupRound, Stage.beforeMeasureRound]; + } + + "Runs GC before each iteration." + shared new beforeEachIteration { + stageSet = [Stage.beforeWarmupIteration, Stage.beforeMeasureIteration]; + } + + + "Runs GC if `stage` satisfies the strategy requirements." + shared actual void gc( "Current execution stage." Stage stage ) { + if ( stageSet.contains( stage ) ) { + System.gc(); + } + } + +} diff --git a/source/herd/asynctest/benchmark/IntegerComparator.ceylon b/source/herd/asynctest/benchmark/IntegerComparator.ceylon new file mode 100644 index 0000000..a316217 --- /dev/null +++ b/source/herd/asynctest/benchmark/IntegerComparator.ceylon @@ -0,0 +1,19 @@ +import java.util { + Comparator +} + + +"Java comparator of Ceylon `Integer`." +since( "0.7.0" ) by( "Lis" ) +class IntegerComparator() satisfies Comparator { + + shared actual Integer compare( Integer first, Integer second ) + => switch ( first <=> second ) + case ( equal ) 0 + case ( larger ) 1 + case ( smaller ) -1; + + + shared actual Boolean equals( Object that ) => (super of Basic).equals(that); + +} diff --git a/source/herd/asynctest/benchmark/IterationsCriteria.ceylon b/source/herd/asynctest/benchmark/IterationsCriteria.ceylon new file mode 100644 index 0000000..fffe4f0 --- /dev/null +++ b/source/herd/asynctest/benchmark/IterationsCriteria.ceylon @@ -0,0 +1,59 @@ +import java.util.concurrent.atomic { + AtomicLong +} + + +"Continues benchmark iterations while total number of iterations doesn't exceed [[numberOfIterations]]. + Number of iterations is equal to number of calls of [[verify]] after last call of [[reset]]. + So, the benchmark is completed when sum of iterations over all used threads reaches [[numberOfIterations]]. + " +tagged( "Criteria" ) +see( `class Options`, `class LocalIterations` ) +throws( `class AssertionError`, "Total number of benchmark iterations is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class TotalIterations ( + "Total number of benchmark iterations. Has to be > 0." Integer numberOfIterations +) + satisfies CompletionCriterion +{ + + "Total number of benchmark iterations has to be > 0." + assert( numberOfIterations > 0 ); + + AtomicLong counter = AtomicLong( 1 ); + + shared actual void reset() { + counter.set( 1 ); + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + return counter.incrementAndGet() > numberOfIterations; + } + +} + + +"Continues benchmark iterations while local (relative to thread) number of iterations doesn't exceed [[numberOfIterations]]. + Number of iterations is equal to number of calls of [[verify]] after last call of [[reset]]." +tagged( "Criteria" ) +see( `class Options`, `class TotalIterations` ) +throws( `class AssertionError`, "Thread local number of benchmark iterations is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class LocalIterations ( + "Thread local number of benchmark iterations. Has to be > 0." Integer numberOfIterations +) + satisfies CompletionCriterion +{ + + "Thread local number of benchmark iterations has to be > 0." + assert( numberOfIterations > 0 ); + + + shared actual void reset() { + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + return result.size >= numberOfIterations; + } + +} diff --git a/source/herd/asynctest/benchmark/Options.ceylon b/source/herd/asynctest/benchmark/Options.ceylon new file mode 100644 index 0000000..971fea5 --- /dev/null +++ b/source/herd/asynctest/benchmark/Options.ceylon @@ -0,0 +1,21 @@ + + +"Benchmark options." +tagged( "Options" ) +see( `function benchmark`, `interface Bench` ) +since( "0.7.0" ) by( "Lis" ) +shared class Options ( + "Criterion which identifies when measure round have to be completed." + shared CompletionCriterion measureCriterion, + "Optional criterion which identifies when warmup round have to be completed. + Warmup round is skipped if `null`." + shared CompletionCriterion? warmupCriterion = null, + "Time unit the results have to be reported with. Default is seconds." + shared TimeUnit timeUnit = TimeUnit.seconds, + "Clock to measure time intervals. Default wall clock." + shared Clock clock = Clock.wall, + "Identifies GC execution strategy, i.e. how often to run GC. + By default GC is executed before warmup round and before iteraton round." + shared GCStrategy gcStrategy = GCStagedStrategy.beforeRounds +) { +} diff --git a/source/herd/asynctest/benchmark/ParameterResult.ceylon b/source/herd/asynctest/benchmark/ParameterResult.ceylon new file mode 100644 index 0000000..6ea46ef --- /dev/null +++ b/source/herd/asynctest/benchmark/ParameterResult.ceylon @@ -0,0 +1,115 @@ + + +"Benchmark run result for a given [[parameter]] and a set of benches." +tagged( "Result" ) +see( `class Result`, `interface Bench` ) +since( "0.7.0" ) by( "Lis" ) +shared final class ParameterResult ( + "Parameter the test has been run with." shared Parameter parameter, + "Results of the run for the given bench." Map, StatisticSummary> byBenches +) + satisfies Map, StatisticSummary> + given Parameter satisfies Anything[] +{ + + variable Float? memoizedSlowestMean = null; + variable Float? memoizedFastestMean = null; + + Float findSlowestMean() { + Float minVal = byBenches.fold( infinity ) ( + ( Float res, ->StatisticSummary> item ) + => if ( item.item.mean < res ) then item.item.mean else res + ); + memoizedSlowestMean = minVal; + return minVal; + } + + Float findFastestMean() { + Float maxVal = byBenches.fold( -infinity ) ( + ( Float res, ->StatisticSummary> item ) + => if ( item.item.mean > res ) then item.item.mean else res + ); + memoizedFastestMean = maxVal; + return maxVal; + } + + + "`true` if item mean equals to `slowestMean`." + Boolean filterBySlowest( ->StatisticSummary> item ) => item.item.mean == slowestMean; + + "`true` if item mean equals to `fastestMean`." + Boolean filterByFastest( ->StatisticSummary> item ) => item.item.mean == fastestMean; + + + "Mean value of the slowest benches." + see( `value slowest` ) + shared Float slowestMean => memoizedSlowestMean else ( memoizedSlowestMean = findSlowestMean() ); + + "A list of benches which showed the smallest performance, i.e. mean number of operations per second is smallest." + see( `value slowestMean` ) + shared {->StatisticSummary>*} slowest => byBenches.filter( filterBySlowest ); + + "Mean value of the fastest benches." + see( `value fastest` ) + shared Float fastestMean => memoizedFastestMean else ( memoizedFastestMean = findFastestMean() ); + + "A list of benches which showed the fastest performance, i.e. mean number of operations per second is largest." + see( `value fastestMean` ) + shared {->StatisticSummary>*} fastest => byBenches.filter( filterByFastest ); + + "Returns map of [[ComparativeStatistic]] related to the given [[benchOrMean]]." + see( `function relativeToSlowest`, `function relativeToFastest` ) + shared Map, ComparativeStatistic> relativeTo ( + "Bench or summary statistic to calculate comparative statistic to." Bench | Float benchOrMean + ) { + if ( exists stat = if ( is Float benchOrMean ) then benchOrMean else get( benchOrMean )?.mean ) { + return byBenches.mapItems ( + ( Bench key, StatisticSummary item ) => ComparativeStatistic( item, stat ) + ); + } + else { + return emptyMap; + } + } + + "Returns map of [[ComparativeStatistic]] related to the slowest bench." + see( `function relativeTo`, `function relativeToFastest` ) + shared Map, ComparativeStatistic> relativeToSlowest() => relativeTo( slowestMean ); + + "Returns map of [[ComparativeStatistic]] related to the fastest bench." + see( `function relativeToSlowest`, `function relativeTo` ) + shared Map, ComparativeStatistic> relativeToFastest() => relativeTo( fastestMean ); + + + shared actual Boolean defines( Object key ) => byBenches.defines( key ); + + shared actual StatisticSummary? get( Object key ) => byBenches.get( key ); + + shared actual StatisticSummary|Default getOrDefault( Object key, Default default ) + => byBenches.getOrDefault( key, default ); + + shared actual Iterator->StatisticSummary> iterator() => byBenches.iterator(); + + shared actual Integer size => byBenches.size; + + shared actual Boolean empty => byBenches.size == 0; + + shared actual ParameterResult clone() => this; + + shared actual Boolean contains( Object entry ) => byBenches.contains( entry ); + + shared actual ParameterResult filterKeys( Boolean(Bench) filtering ) + => ParameterResult( parameter, byBenches.filterKeys( filtering ) ); + + shared actual Integer hash => 37 + byBenches.hash; + + shared actual Boolean equals( Object that ) { + if ( is ParameterResult that ) { + return byBenches == that.byBenches; + } + else { + return false; + } + } + +} diff --git a/source/herd/asynctest/benchmark/Result.ceylon b/source/herd/asynctest/benchmark/Result.ceylon new file mode 100644 index 0000000..e1f1a78 --- /dev/null +++ b/source/herd/asynctest/benchmark/Result.ceylon @@ -0,0 +1,118 @@ +import ceylon.collection { + HashMap +} + + +"Benchmark run result for a set of parameters and benches." +tagged( "Result" ) +see( `class ParameterResult` ) +since( "0.7.0" ) by( "Lis" ) +shared final class Result ( + "Time unit the time has been measured with." + shared TimeUnit timeUnit, + "Results for a given parameter value." + Map> byParameter +) + satisfies Map> + given Parameter satisfies Anything[] +{ + + class BenchMap( Bench bench ) satisfies Map { + + Map> outerParameters => byParameter; + + shared actual Boolean defines( Object key ) => byParameter.get( bench )?.defines( key ) else false; + + shared actual StatisticSummary? get( Object key ) => byParameter.get( bench )?.get( key ); + + shared actual StatisticSummary|Default getOrDefault( Object key, Default default ) + => byParameter.get( key )?.get( key ) else default; + + shared actual IteratorStatisticSummary> iterator() + => object satisfies IteratorStatisticSummary> { + IteratorParameterResult> paramIterator = byParameter.iterator(); + + shared actual StatisticSummary>|Finished next() { + if ( is ParameterResult> nn = paramIterator.next() ) { + "Results don't contain specified bench." + assert( exists ret = nn.item.get( bench ) ); + return Entry( nn.key, ret ); + } + else { + return finished; + } + } + }; + + shared actual Integer size => byParameter.size; + + shared actual Boolean empty => byParameter.size == 0; + + shared actual BenchMap clone() => this; + + shared actual Boolean contains( Object entry ) => byParameter.get( bench )?.contains( entry ) else false; + + shared actual Integer hash => 37 * bench.hash + byParameter.hash; + + shared actual Boolean equals( Object that ) { + if ( is BenchMap that ) { + return bench == that.bench && outerParameters == that.outerParameters; + } + else { + return false; + } + } + + } + + HashMap, BenchMap> benches = HashMap, BenchMap>(); + + + "Returns results for a specific bench." + shared Map forBench( "Bench the results are asked for." Bench bench ) { + if ( exists ret = benches.get( bench ) ) { + return ret; + } + else if ( exists f = byParameter.first, f.item.defines( bench ) ){ + BenchMap m = BenchMap( bench ); + benches.put( bench, m ); + return m; + } + else { + return emptyMap; + } + } + + + shared actual Boolean defines( Object key ) => byParameter.defines( key ); + + shared actual ParameterResult? get( Object key ) => byParameter.get( key ); + + shared actual ParameterResult|Default getOrDefault( Object key, Default default ) + => byParameter.getOrDefault( key, default ); + + shared actual IteratorParameterResult> iterator() => byParameter.iterator(); + + shared actual Integer size => byParameter.size; + + shared actual Boolean empty => byParameter.size == 0; + + shared actual Result clone() => this; + + shared actual Boolean contains( Object entry ) => byParameter.contains( entry ); + + shared actual Result filterKeys( Boolean(Parameter) filtering ) + => Result( timeUnit, byParameter.filterKeys( filtering ) ); + + shared actual Integer hash => 37 + byParameter.hash; + + shared actual Boolean equals( Object that ) { + if ( is Result that ) { + return byParameter == that.byParameter; + } + else { + return false; + } + } + +} diff --git a/source/herd/asynctest/benchmark/SingleBench.ceylon b/source/herd/asynctest/benchmark/SingleBench.ceylon new file mode 100644 index 0000000..153fc4d --- /dev/null +++ b/source/herd/asynctest/benchmark/SingleBench.ceylon @@ -0,0 +1,25 @@ + +"Executes test function in a single thread." +tagged( "Bench" ) +see( `function benchmark`, `class EmptyParameterBench` ) +since( "0.7.0" ) by( "Lis" ) +shared final class SingleBench ( + "Bench title. Generally unique." + String title, + "Function to be tested." + shared actual Anything(*Parameter) runIteration +) + extends BaseBench( title ) + given Parameter satisfies Anything[] +{ + + shared actual Boolean equals( Object other ) { + if ( is SingleBench other ) { + return title == other.title; + } + else { + return false; + } + } + +} diff --git a/source/herd/asynctest/benchmark/Stage.ceylon b/source/herd/asynctest/benchmark/Stage.ceylon new file mode 100644 index 0000000..c8e00a3 --- /dev/null +++ b/source/herd/asynctest/benchmark/Stage.ceylon @@ -0,0 +1,53 @@ + + +"Identifies a stage of the bench execution." +see( `class Options`, `interface GCStrategy`, `class GCStagedStrategy` ) +tagged( "Options" ) +since( "0.7.0" ) by( "Lis" ) +shared class Stage + of beforeWarmupRound | afterWarmupRound | beforeWarmupIteration | afterWarmupIteration + | beforeMeasureRound | afterMeasureRound | beforeMeasureIteration | afterMeasureIteration +{ + shared actual String string; + + "Before warmup round." + shared new beforeWarmupRound { + string = "before warmup round stage"; + } + + "After warmup round." + shared new afterWarmupRound { + string = "after warmup round stage"; + } + + "Before warmup iteration." + shared new beforeWarmupIteration { + string = "before warmup iteration stage"; + } + + "After warmup iteration." + shared new afterWarmupIteration { + string = "after warmup iteration stage"; + } + + "Before measure round." + shared new beforeMeasureRound { + string = "before measure round stage"; + } + + "After measure round." + shared new afterMeasureRound { + string = "after measure round stage"; + } + + "Before measure iteration." + shared new beforeMeasureIteration { + string = "before measure iteration stage"; + } + + "After measure iteration." + shared new afterMeasureIteration { + string = "after measure iteration stage"; + } + +} diff --git a/source/herd/asynctest/benchmark/StatisticSummary.ceylon b/source/herd/asynctest/benchmark/StatisticSummary.ceylon new file mode 100644 index 0000000..68af0aa --- /dev/null +++ b/source/herd/asynctest/benchmark/StatisticSummary.ceylon @@ -0,0 +1,106 @@ +import java.lang { + Math +} + + +"Statistic summary for a stream of variate values." +tagged( "Result" ) +see( `class ParameterResult` ) +since( "0.6.0" ) by( "Lis" ) +shared final class StatisticSummary { + + "Minimum of the values that have been statisticaly treated." + shared Float min; + "Maximum of the values that have been statisticaly treated." + shared Float max; + "Mean value." + shared Float mean; + "Returns standard deviation of the values that have been statisticaly treated. + Standard deviation is `variance^0.5`." + shared Float standardDeviation; + "The number of the values that have been statisticaly treated." + shared Integer size; + + "Creates new statistic summary with the given data." + shared new ( + "Minimum of the values that have been statisticaly treated." + Float min, + "Maximum of the values that have been statisticaly treated." + Float max, + "Mean value." + Float mean, + "Returns standard deviation of the values that have been statisticaly treated. + Standard deviation is `variance^0.5`." + Float standardDeviation, + "The number of the values that have been statisticaly treated." + Integer size + ) { + this.min = min; + this.max = max; + this.mean = mean; + this.standardDeviation = standardDeviation; + this.size = size; + } + + "Creates new statistic summary by combining two others." + shared new combined( StatisticSummary first, StatisticSummary second ) { + this.min = first.min < second.min then first.min else second.min; + this.max = first.max > second.max then first.max else second.max; + this.size = first.size + second.size; + Float firstRatio = first.size.float / size; + Float secondSize = second.size.float; + Float secondRatio = secondSize / size; + this.mean = first.mean * firstRatio + second.mean * secondRatio; + Float delta = second.mean - first.mean; + Float m2 = first.variance * ( first.size - 1 ) + second.variance * ( second.size - 1 ) + + delta * delta * firstRatio * secondSize; + Float variance => if ( size > 1 ) then m2 / ( size - 1 ) else 0.0; + this.standardDeviation = Math.sqrt( variance ); + } + + + "Variance of the values that have been statisticaly treated. + The variance is mean((x-mean(x))^2)." + shared Float variance => standardDeviation * standardDeviation; + + "Sample variance is size/(size - 1)*variance." + shared Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; + + "Sample standard deviation is sqrt(size/(size - 1)*variance)." + shared Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) ) * standardDeviation else 0.0; + + "Equals to standardDeviation/sqrt(size)." + shared Float standardError => if ( size > 1 ) then standardDeviation / Math.sqrt( size.float ) else 0.0; + + "Equals to sampleStandardDeviation/sqrt(size)." + shared Float sampleError => if ( size > 1 ) then standardDeviation / Math.sqrt( ( size - 1 ).float ) else 0.0; + + "Equals to sampleError/mean." + shared Float relativeSampleError => sampleError / mean; + + + shared actual Boolean equals( Object other ) { + if ( is StatisticSummary other ) { + return min == other.min && max == other.max + && mean == other.mean && standardDeviation == other.standardDeviation + && size == other.size; + } + else { + return false; + } + } + + shared actual Integer hash { + variable Integer ret = 1; + ret = min.hash + 37 * ret; + ret = max.hash + 37 * ret; + ret = mean.hash + 37 * ret; + ret = standardDeviation.hash + 37 * ret; + ret = size + 37 * ret; + return ret; + } + + + string => "mean=``Float.format(mean, 0, 3)``, standard deviation=``Float.format(standardDeviation, 0, 3)``, " + + "max=``Float.format(max, 0, 3)``, min=``Float.format(min, 0, 3)``, total samples=``size``"; +} diff --git a/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon b/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon new file mode 100644 index 0000000..f2ef961 --- /dev/null +++ b/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon @@ -0,0 +1,111 @@ +import java.lang.management { + ManagementFactory +} + + +"Represents clock local to thread - has to be executed from a one thread." +since( "0.7.0" ) by( "Lis" ) +interface ThreadLocalClock { + + "Initializes the clock." + shared formal void initialize(); + + "Starts time interval measure from the current time." + shared formal void start(); + + "Measures time interval from the last [[start]] call and up to now." + shared formal Integer measure(); +} + + +"Wall clock uses `system.nanoseconds` to measure time interval." +since( "0.7.0" ) by( "Lis" ) +class WallLocalClock() satisfies ThreadLocalClock { + + variable Integer startWallTime = system.nanoseconds; + + shared actual void start() { + startWallTime = system.nanoseconds; + } + + shared actual void initialize() {} + + shared actual Integer measure() => system.nanoseconds - startWallTime; +} + + +"Base for the CPU time interval measurements. + Calculates a factor to scale wall time interval corresponds to measure request. + The factor is moving averaged ratio of CPU time interval to wall time interval. + Ends of CPU time interval are measured as close as possible to ends of wall time interval. + + [[readCurrentTime]] provides particular implementation of CPU or user times. + " +since( "0.7.0" ) by( "Lis" ) +abstract class ThreadClock() satisfies ThreadLocalClock { + + variable Integer startWallTime = system.nanoseconds; + variable Integer lastWallTime = 0; + variable Integer lastSample = 0; + variable Float factor = 1.0; + + "Returns current time based on strategy." + shared formal Integer readCurrentTime(); + + + shared actual void initialize() { + value threadMXBean = ManagementFactory.threadMXBean; + if ( !threadMXBean.threadCpuTimeEnabled ) { + threadMXBean.threadCpuTimeEnabled = true; + } + lastSample = readCurrentTime(); + startWallTime = system.nanoseconds; + lastWallTime = startWallTime; + factor = 1.0; + } + + shared actual void start() { + Integer currentSample = readCurrentTime(); + Integer currentWallTime = system.nanoseconds; + if ( currentSample > lastSample && currentWallTime > lastWallTime ) { + // Moving averaging factor! + factor = 0.9 * factor + 0.1 * ( currentSample - lastSample ) / ( currentWallTime - lastWallTime ); + lastSample = currentSample; + lastWallTime = currentWallTime; + } + startWallTime = system.nanoseconds; + } + + shared actual Integer measure() => ( factor * ( system.nanoseconds - startWallTime ) ).integer; + +} + + +"CPU clock uses `ThreadMXBean.currentThreadCpuTime` to measure time interval." +since( "0.7.0" ) by( "Lis" ) +class CPULocalClock() extends ThreadClock() { + shared actual Integer readCurrentTime() { + value threadMXBean = ManagementFactory.threadMXBean; + if ( threadMXBean.currentThreadCpuTimeSupported ) { + return threadMXBean.currentThreadCpuTime; + } + else { + return system.nanoseconds; + } + } +} + + +"User clock uses `ThreadMXBean.currentThreadUserTime` to measure time interval." +since( "0.7.0" ) by( "Lis" ) +class UserLocalClock() extends ThreadClock() { + shared actual Integer readCurrentTime() { + value threadMXBean = ManagementFactory.threadMXBean; + if ( threadMXBean.currentThreadCpuTimeSupported ) { + return threadMXBean.currentThreadUserTime; + } + else { + return system.nanoseconds; + } + } +} diff --git a/source/herd/asynctest/benchmark/TimeCriteria.ceylon b/source/herd/asynctest/benchmark/TimeCriteria.ceylon new file mode 100644 index 0000000..b207cdc --- /dev/null +++ b/source/herd/asynctest/benchmark/TimeCriteria.ceylon @@ -0,0 +1,80 @@ +import java.util.concurrent.atomic { + AtomicLong +} +import java.lang { + Thread +} +import java.util.concurrent { + ConcurrentSkipListMap +} + + +"Continues benchmark iterations while overall time accumulated by benchmark iterations doesn't exceed + [[totalTime]]." +tagged( "Criteria" ) +see( `class Options`, `class LocalBenchTime` ) +throws( `class AssertionError`, "Total benchmark time is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class TotalBenchTime ( + "Maximum time (in [[timeUnit]]) to be accumulated by benchmark iterations." Integer totalTime, + "Unit the [[totalTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds +) + satisfies CompletionCriterion +{ + "Total benchmark time has to be > 0." + assert( totalTime > 0 ); + Integer totalNanoseconds = ( totalTime * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + + AtomicLong accumulatedTime = AtomicLong( 0 ); + + + shared actual void reset() { + accumulatedTime.set( 0 ); + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit deltaTimeUnit ) { + Integer timeToAdd = ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + return accumulatedTime.addAndGet( timeToAdd ) > totalNanoseconds; + } + +} + + +"Continues benchmark iterations while local (relative to thread) time accumulated by benchmark iterations doesn't exceed + [[localTime]]." +tagged( "Criteria" ) +see( `class Options`, `class TotalBenchTime` ) +throws( `class AssertionError`, "Local benchmark time is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class LocalBenchTime ( + "Maximum time (in [[timeUnit]]) to be accumulated by local (relative to thread) iterations." Integer localTime, + "Unit the [[localTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds +) + satisfies CompletionCriterion +{ + + "Thread local benchmark time has to be > 0." + assert( localTime > 0 ); + Integer nanoseconds = ( localTime * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + + ConcurrentSkipListMap localAccumulators = ConcurrentSkipListMap( IntegerComparator() ); + + + shared actual void reset() { + localAccumulators.clear(); + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit deltaTimeUnit ) { + Integer id = Thread.currentThread().id; + Integer timeVal; + if ( exists current = localAccumulators.get( id ) ) { + timeVal = current + ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + } + else { + timeVal = ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + } + localAccumulators.put( id, timeVal ); + return timeVal > nanoseconds; + } + +} diff --git a/source/herd/asynctest/benchmark/TimeUnit.ceylon b/source/herd/asynctest/benchmark/TimeUnit.ceylon new file mode 100644 index 0000000..f0c2651 --- /dev/null +++ b/source/herd/asynctest/benchmark/TimeUnit.ceylon @@ -0,0 +1,61 @@ + + +"Specifies time unit." +tagged( "Options" ) +see( `class Options`, `class Result` ) +since( "0.7.0" ) by( "Lis" ) +shared class TimeUnit + of seconds | milliseconds | microseconds | nanoseconds | minutes | hours +{ + + "Factor to seconds." + shared Float factorToSeconds; + + "Short designation." + shared String shortString; + + shared actual String string; + + "Seconds." + shared new seconds { + factorToSeconds = 1.0; + shortString = "s"; + string = "seconds"; + } + + "Milliseconds." + shared new milliseconds { + factorToSeconds = 1.0e-3; + shortString = "ms"; + string = "milliseconds"; + } + + "Microseconds." + shared new microseconds { + factorToSeconds = 1.0e-6; + shortString = "micros"; + string = "microseconds"; + } + + "Nanoseconds." + shared new nanoseconds { + factorToSeconds = 1.0e-9; + shortString = "ns"; + string = "nanoseconds"; + } + + "Minutes." + shared new minutes { + factorToSeconds = 60.0; + shortString = "m"; + string = "minutes"; + } + + "Hours." + shared new hours { + factorToSeconds = 3600.0; + shortString = "h"; + string = "hours"; + } + +} diff --git a/source/herd/asynctest/benchmark/benchmark.ceylon b/source/herd/asynctest/benchmark/benchmark.ceylon new file mode 100644 index 0000000..e7ede41 --- /dev/null +++ b/source/herd/asynctest/benchmark/benchmark.ceylon @@ -0,0 +1,74 @@ +import ceylon.collection { + HashMap +} + + +"Runs benchmark testing. + Executes each given bench with each given parameter. + Bench is responsible for the execution details and calculation performance statistic." +throws( `class AssertionError`, "number of measure rounds is <= 0" ) +since( "0.7.0" ) by( "Lis" ) +shared Result benchmark ( + "Benchmark options of benches executing." + Options options, + "A list of benches to be executed." + Bench[] benches, + "A list of parameters the test has to be executed with." + Parameter* parameters +) + given Parameter satisfies Anything[] +{ + if ( parameters nonempty ) { + // run test for each parameter and each bench + HashMap> res = HashMap>(); + for ( param in parameters ) { + if ( !res.defines( param ) ) { + HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary>(); + for ( bench in benches ) { + if ( !benchRes.defines( bench ) ) { + // initialize black hole + blackHole.clear(); + // initialize criteria + options.measureCriterion.reset(); + options.warmupCriterion?.reset(); + // execute bench + value br = bench.execute( options, param ); + // store results + benchRes.put( bench, br ); + // avoid dead code elimination for the black hole + blackHole.verifyNumbers(); + } + } + res.put( param, ParameterResult( param, benchRes ) ); + } + } + return Result( options.timeUnit, res ); + } + else { + // parameter list is empty! only empty parameters test is admited + "benchmarking: if parameter list is empty, `Parameter` has to be `empty`." + assert ( [] is Parameter ); + assert ( is Bench<[]>[] benches ); + + HashMap<[], ParameterResult<[]>> res = HashMap<[], ParameterResult<[]>>(); + HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary>(); + for ( bench in benches ) { + if ( !benchRes.defines( bench ) ) { + // initialize black hole + blackHole.clear(); + // initialize criteria + options.measureCriterion.reset(); + options.warmupCriterion?.reset(); + // execute bench + value br = bench.execute( options, [] ); + // store results + benchRes.put( bench, br ); + // avoid dead code elimination for the black hole + blackHole.verifyNumbers(); + } + } + res.put( [], ParameterResult<[]>( [], benchRes ) ); + assert ( is Result ret = Result<[]>( options.timeUnit, res ) ); + return ret; + } +} diff --git a/source/herd/asynctest/benchmark/blackHole.ceylon b/source/herd/asynctest/benchmark/blackHole.ceylon new file mode 100644 index 0000000..09570cc --- /dev/null +++ b/source/herd/asynctest/benchmark/blackHole.ceylon @@ -0,0 +1,49 @@ +import java.util.concurrent.atomic { + AtomicLong +} + + +"Represents a black hole in order to eliminate JIT optimizations. + + Usage: + * call [[clear]] before testing + * call [[consume]] when object has to be consumed to eliminate JIT optimizations + * call [[verifyNumbers]] after the test in order to eliminate JIT optimizations on this `blackHole` object + " +since( "0.7.0" ) by( "Lis" ) +object blackHole { + + "Total number of non-null objects consumed by the black hole." + shared AtomicLong totalObjects = AtomicLong( 0 ); + "Total number of nulls consumed by the black hole." + shared AtomicLong totalNulls = AtomicLong( 0 ); + + "Prevents JIT to eliminate dependent computations." + shared void consume( Anything something ) { + if ( something exists ) { + totalObjects.incrementAndGet(); + } + else { + totalNulls.incrementAndGet(); + } + } + + "Resets number of consumed objects to zero." + shared void clear() { + totalObjects.set( 0 ); + totalNulls.set( 0 ); + } + + + "Verifies number of consumed objects in order to eliminate JIT optimizations on this `blackHole` object." + shared void verifyNumbers() { + "Total number of consumed objects may not be less than zero." + assert ( totalObjects.get() + totalNulls.get() >= 0 ); + } + +} + + +"Prevents JIT to eliminate dependent computations." +since( "0.7.0" ) by( "Lis" ) +shared void pushToBlackHole( Anything something ) => blackHole.consume( something ); diff --git a/source/herd/asynctest/benchmark/contextWriters.ceylon b/source/herd/asynctest/benchmark/contextWriters.ceylon new file mode 100644 index 0000000..89e6252 --- /dev/null +++ b/source/herd/asynctest/benchmark/contextWriters.ceylon @@ -0,0 +1,120 @@ +import herd.asynctest { + AsyncTestContext +} +import herd.asynctest.internal { + stringify +} + + +"Helper to represent parameter." +since( "0.7.0" ) by( "Lis" ) +String? stringifyParameter( Parameter param ) + given Parameter satisfies Anything[] + => if ( param.size > 1 ) then stringify( param ) else + if ( exists f = param.first ) then stringify( param.first ) else null; + + +"Writes absolute results of the benchmarking to the context in format: + `bench.title` with `parameter`: mean=`mean value`, dev=`sample deviation value`; error=`sample error value`% + " +tagged( "Writer" ) +see( `function benchmark`, `function writeRelativeToSlowest`, `function writeRelativeToFastest` ) +since( "0.7.0" ) by( "Lis" ) +shared void writeAbsolute( AsyncTestContext context, Result results ) + given Parameter satisfies Anything[] +{ + String tuShort = " op/" + results.timeUnit.shortString; + for ( param->paramRes in results ) { + if ( exists paramStr = stringifyParameter( param ) ) { + for ( bench->stat in paramRes ) { + context.succeed ( + "``bench.title`` with `" + paramStr + + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + ); + } + } + else { + for ( bench->stat in paramRes ) { + context.succeed ( + "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + ); + } + } + } +} + + +"Writes relative to the slowest results of the benchmarking to the context in format: + `bench.title` with `parameter`: mean=`mean value`, dev=`sample deviation value`, `relative mean value`% to slowest; error=`sample error value`% + " +tagged( "Writer" ) +see( `function benchmark`, `function writeRelativeToFastest`, `function writeAbsolute` ) +since( "0.7.0" ) by( "Lis" ) +shared void writeRelativeToSlowest( AsyncTestContext context, Result results ) + given Parameter satisfies Anything[] +{ + String tuShort = " op/" + results.timeUnit.shortString; + for ( param->paramRes in results ) { + if ( exists paramStr = stringifyParameter( param ) ) { + for ( bench->stat in paramRes.relativeToSlowest() ) { + context.succeed ( + "``bench.title`` with `" + paramStr + + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + + "``stat.relativeMean``% to slowest; " + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + ); + } + } + else { + for ( bench->stat in paramRes.relativeToSlowest() ) { + context.succeed ( + "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + + "``stat.relativeMean``% to slowest; " + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + ); + } + } + } +} + + +"Writes relative to the fastest results of the benchmarking to the context in format: + `bench.title` with `parameter`: mean=`mean value`; dev=`sample deviation value`; `relative mean value`% to fastest; error=`sample error value`% + " +tagged( "Writer" ) +see( `function benchmark`, `function writeRelativeToSlowest`, `function writeAbsolute` ) +since( "0.7.0" ) by( "Lis" ) +shared void writeRelativeToFastest( AsyncTestContext context, Result results ) + given Parameter satisfies Anything[] +{ + String tuShort = " op/" + results.timeUnit.shortString; + for ( param->paramRes in results ) { + if ( exists paramStr = stringifyParameter( param ) ) { + for ( bench->stat in paramRes.relativeToFastest() ) { + context.succeed ( + "``bench.title`` with `" + paramStr + + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + + "``stat.relativeMean``% to fastest; " + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + ); + } + } + else { + for ( bench->stat in paramRes.relativeToFastest() ) { + context.succeed ( + "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + + "``stat.relativeMean``% to fastest; " + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + ); + } + } + } +} diff --git a/source/herd/asynctest/benchmark/delegateWriter.ceylon b/source/herd/asynctest/benchmark/delegateWriter.ceylon new file mode 100644 index 0000000..72ea57f --- /dev/null +++ b/source/herd/asynctest/benchmark/delegateWriter.ceylon @@ -0,0 +1,21 @@ +import herd.asynctest { + AsyncTestContext +} + + +"Delegates benchmark run result writing to the given `writers`." +tagged( "Writer" ) +see( `function benchmark`, `function writeRelativeToSlowest`, + `function writeRelativeToFastest`, `function writeAbsolute` ) +since( "0.7.0" ) by( "Lis" ) +shared void delegateWriter ( + "Writers to be delegated to write benchmark run result." + {Anything(AsyncTestContext, Result)*} writers +) + ( AsyncTestContext context, Result results ) + given Parameter satisfies Anything[] +{ + for ( item in writers ) { + item( context, results ); + } +} diff --git a/source/herd/asynctest/benchmark/package.ceylon b/source/herd/asynctest/benchmark/package.ceylon new file mode 100644 index 0000000..ab4eb8d --- /dev/null +++ b/source/herd/asynctest/benchmark/package.ceylon @@ -0,0 +1,121 @@ + +" + Library to run benchmarks. + + + ### Terminology + + * Benchmark is a set of benches with a unified test parameter type and a set of the parameter values. + * Bench is an executor of the function to be tested. + * Bench run statistic is number of operations per time unit, i.e. number of test function calls per time unit. + * Benchmark run result is collection of bench run statistic for all given benches and all given test parameter values. + + + ### Bench + + Bench is responsible for the test function execution and calculation performance statistic. + Each bench has to satisfy [[Bench]] interface. + Any particular bench may implement its own testing semantic and purposes. + + For instance, [[SingleBench]] is intended to run test function in a single thread. + + + ### Benchmark execution + + [[benchmark]] is intended to run benchmark. The function executes each given bench with each given parameter + and returns results of benchmark test as instance of [[Result]]. + + + ### JIT optimization + + JIT optimization may eliminate some code at runtime if it finds that the result of the code execution is + not used anywhere. + + Mainly there are two eliminations: + 1. Unused results of calculations + 2. Constants + + + #### Black hole + + In order to avoid unused results elimination results may be passed to black hole using [[pushToBlackHole]] function. + + Suppose we would like to test plus operation: + + void plusBenchmark() { + value x = 2; + value y = 3; + value z = x + y; + } + + This function call might be eliminated at all, since it results to nothing. To avoid this black hole might be used: + + void plusBenchmark() { + value x = 2; + value y = 3; + pushToBlackHole(x + y); + } + + All returned values are pushed to black hole by default. So, the above example might be replaced with: + + Integer plusBenchmark() { + value x = 2; + value y = 3; + return x + y; + } + + + #### Constants + + There are two constants `x` and `y` in the above examples. JIT may find that result of `+` operation is always + the same and replace the last function with + + Integer plusBenchmark() { + return 5; + } + + So, the plus operation will never be executed. In order to avoid this the parameters might be passed to function as arguments + or declared outside the function scope as variables: + + variable Integer x = 2; + variable Integer y = 3; + + Integer plusBenchmark() { + return x + y; + } + + > Note: returning value is prefer to direct pushing to black hole, since in this case time consumed by black hole is excluded + from total time of the test function execution. + + + ### Writing benchmark result + + Benchmark run results may be writed to `AsyncTestContext` using a number of writers. See functions tagged as `Writer`. + + + ### Example + + Integer plusBenchmarkFunction(Integer x, Integer y) { + return x + y; + } + Integer minusBenchmarkFunction(Integer x, Integer y) { + return x - y; + } + + shared test async void plusMinusBenchmark(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options(LocalIterations(20000).or(LocalError(0.002))), + [SingleBench(\"plus\", plusBenchmarkFunction), + SingleBench(\"minus\", minusBenchmarkFunction)], + [1, 1], [2, 3], [25, 34] + ) + ); + context.complete(); + } + + + " +since( "0.7.0" ) by( "Lis" ) +shared package herd.asynctest.benchmark; diff --git a/source/herd/asynctest/benchmark/stringifyNumberOfOperations.ceylon b/source/herd/asynctest/benchmark/stringifyNumberOfOperations.ceylon new file mode 100644 index 0000000..0c29e50 --- /dev/null +++ b/source/herd/asynctest/benchmark/stringifyNumberOfOperations.ceylon @@ -0,0 +1,12 @@ + + +"Returns string representation of number of operations" +since( "0.7.0" ) by( "Lis" ) +String stringifyNumberOfOperations( Float numOfOp ) { + if ( numOfOp > 1.0 ) { + return numOfOp.integer.string; + } + else { + return Float.format( numOfOp, 0, 2 ); + } +} diff --git a/source/herd/asynctest/exceptions.ceylon b/source/herd/asynctest/exceptions.ceylon index 87a2dcf..b6e505e 100644 --- a/source/herd/asynctest/exceptions.ceylon +++ b/source/herd/asynctest/exceptions.ceylon @@ -25,7 +25,7 @@ shared class FactoryReturnsNothing( String factoryTitle ) "Thrown when instantiated class has neither default constructor nor factory function." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared class IncompatibleInstantiation ( "Declaration of the trying to instantiate class." shared ClassDeclaration declaration @@ -38,7 +38,7 @@ shared class IncompatibleInstantiation ( "Collects multiple abort reasons in a one exception." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared class MultipleAbortException ( "The collected exceptions." shared Throwable[] abortReasons, "A brief abort description" shared String description = "multiple abort reasons (``abortReasons.size``):" diff --git a/source/herd/asynctest/internal/MarkerThreadGroup.ceylon b/source/herd/asynctest/internal/MarkerThreadGroup.ceylon index 071cea2..be3ff6c 100644 --- a/source/herd/asynctest/internal/MarkerThreadGroup.ceylon +++ b/source/herd/asynctest/internal/MarkerThreadGroup.ceylon @@ -3,7 +3,7 @@ import java.lang { } "Just a marker for the thread group - means the group is test executor group." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared abstract class MarkerThreadGroup( String title ) extends ThreadGroup( title ) { "ID of the currently run test." shared formal Integer testID; diff --git a/source/herd/asynctest/internal/SimpleStat.ceylon b/source/herd/asynctest/internal/SimpleStat.ceylon new file mode 100644 index 0000000..c66b528 --- /dev/null +++ b/source/herd/asynctest/internal/SimpleStat.ceylon @@ -0,0 +1,65 @@ +import java.lang { + Math +} +import herd.asynctest.benchmark { + StatisticSummary +} + + +"Provides thread-unsafe statistic calculations." +since( "0.7.0" ) by( "Lis" ) +shared class SimpleStat() { + + variable Float minVal = infinity; + variable Float maxVal = -infinity; + variable Float meanVal = 0.0; + variable Float m2Val = 0.0; + variable Integer sizeVal = 0; + + "Minimum of the values that have been statisticaly treated." + shared Float min => minVal; + "Maximum of the values that have been statisticaly treated." + shared Float max => maxVal; + "Mean calculated within Welford's method of standard deviation computation." + shared Float mean => meanVal; + "Second moment used in Welford's method of standard deviation computation." + shared Float m2 => m2Val; + "The number of the values that have been statisticaly treated." + shared Integer size => sizeVal; + + "Returns variance of the values that have been statisticaly treated. + The variance is mean((x-mean(x))^2)." + shared Float variance => if ( sizeVal > 1 ) then m2Val / ( sizeVal - 1 ) else 0.0; + + "Returns standard deviation of the values that have been statisticaly treated. + Standard deviation is `variance^0.5`." + shared Float standardDeviation => Math.sqrt( variance ); + + "Sample variance is size/(size - 1)*variance." + shared Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; + + "Sample standard deviation is sqrt(size/(size - 1)*variance)." + shared Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) * variance ) else 0.0; + + "Equals to standardDeviation/sqrt(size)." + shared Float standardError => if ( size > 1 ) then Math.sqrt( variance / size.float ) else 0.0; + + "Equals to sampleStandardDeviation/sqrt(size)." + shared Float sampleError => if ( size > 1 ) then Math.sqrt( variance / ( size - 1 ).float ) else 0.0; + + "Equals to sampleError/mean." + shared Float relativeSampleError => sampleError / mean; + + + shared void sample( Float val ) { + minVal = val < minVal then val else minVal; + maxVal = val > maxVal then val else maxVal; + Float delta = val - meanVal; + sizeVal ++; + meanVal = meanVal + delta / sizeVal; + m2Val = sizeVal > 1 then m2Val + delta * ( val - meanVal ) else 0.0; + } + + shared StatisticSummary result => StatisticSummary( minVal, maxVal, meanVal, standardDeviation, sizeVal ); + +} diff --git a/source/herd/asynctest/rule/Stat.ceylon b/source/herd/asynctest/internal/StatisticCalculator.ceylon similarity index 81% rename from source/herd/asynctest/rule/Stat.ceylon rename to source/herd/asynctest/internal/StatisticCalculator.ceylon index 7c4bd62..3d0c868 100644 --- a/source/herd/asynctest/rule/Stat.ceylon +++ b/source/herd/asynctest/internal/StatisticCalculator.ceylon @@ -1,38 +1,16 @@ -import java.lang { - Math -} import java.util.concurrent.atomic { AtomicReference } - - -"Statistic summary for a stream of variate values." -see( `class StatisticRule`, `class MeterRule` ) -by( "Lis" ) since( "0.6.0" ) -shared class StatisticSummary ( - "Minimum of the values that have been statisticaly treated." - shared Float min, - "Maximum of the values that have been statisticaly treated." - shared Float max, - "Mean value." - shared Float mean, - "Returns standard deviation of the values that have been statisticaly treated. - Standard deviation is `variance^0.5`." - shared Float standardDeviation, - "The number of the values that have been statisticaly treated." - shared Integer size -) { - "Variance of the values that have been statisticaly treated. - The variance is mean((x-mean(x))^2)." - shared Float variance => standardDeviation * standardDeviation; - - string => "mean=``Float.format(mean, 0, 3)``, standard deviation=``Float.format(standardDeviation, 0, 3)``, " + - "max=``Float.format(max, 0, 3)``, min=``Float.format(min, 0, 3)``, total samples=``size``"; +import herd.asynctest.benchmark { + StatisticSummary +} +import java.lang { + Math } "Current values of statistic calculations." -by( "Lis" ) since( "0.6.1" ) +since( "0.7.0" ) by( "Lis" ) class StatisticStream { "Minimum of the values that have been statisticaly treated." @@ -54,7 +32,7 @@ class StatisticStream { m2 = 0.0; size = 0; } - + "New stream by the given values." shared new withValues( Float* values ) { variable Float min = infinity; @@ -73,7 +51,7 @@ class StatisticStream { m2 += delta * ( item - mean ); } if ( size < 2 ) { m2 = 0.0; } - + this.min = min; this.max = max; this.mean = mean; @@ -99,7 +77,7 @@ class StatisticStream { this.m2 = m2; this.size = size; } - + "New stream with precalculated data." shared new withData( Float min, Float max, Float mean, Float m2, Integer size ) { this.min = min; @@ -135,9 +113,8 @@ class StatisticStream { "Calculates statistic data for stream of variate values." -see( `class StatisticRule`, `class MeterRule` ) -by( "Lis" ) since( "0.6.0" ) -class StatisticCalculator() { +since( "0.6.0" ) by( "Lis" ) +shared class StatisticCalculator() { AtomicReference stat = AtomicReference( StatisticStream.empty() ); diff --git a/source/herd/asynctest/internal/declarationVerifier.ceylon b/source/herd/asynctest/internal/declarationVerifier.ceylon index cefa4cf..f2e224b 100644 --- a/source/herd/asynctest/internal/declarationVerifier.ceylon +++ b/source/herd/asynctest/internal/declarationVerifier.ceylon @@ -22,7 +22,7 @@ import ceylon.test.annotation { "Verifies if declaration meets templates" -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared object declarationVerifier { "Memoization of [[TestRuleAnnotation]]." diff --git a/source/herd/asynctest/match/checkMatchers.ceylon b/source/herd/asynctest/match/checkMatchers.ceylon index de3d822..aeb8230 100644 --- a/source/herd/asynctest/match/checkMatchers.ceylon +++ b/source/herd/asynctest/match/checkMatchers.ceylon @@ -24,7 +24,7 @@ shared class EqualTo ( "Verifies if matching value is _not_ equal to `merit` using operator `!=`." -tagged( "Checkers", "Comparators" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Checkers", "Comparators" ) since( "0.7.0" ) by( "Lis" ) shared class NotEqualTo ( "Value which is expected to be not equal to matching one." Value merit ) @@ -60,7 +60,7 @@ shared class Identical ( "Verifies if matching value is _not_ identical to `merit` using operator `===`." -tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Checkers" ) since( "0.7.0" ) by( "Lis" ) shared class NotIdentical ( "Value which is expected to be _not_ identical to matching one." Value merit ) @@ -142,7 +142,7 @@ shared class EqualWith ( "Verifies if matching value is _not_ equal to [[merit]] using given [[comparator]]." -tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Checkers" ) since( "0.7.0" ) by( "Lis" ) shared class NotEqualWith ( "Value which is expected to be _not_ equal to matching one with the given [[comparator]]." Value merit, @@ -184,7 +184,7 @@ shared class IsType() "Verifies if matching value is _not_ of `Check` type." -tagged( "Checkers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Checkers" ) since( "0.7.0" ) by( "Lis" ) shared class IsNotType() satisfies Matcher { diff --git a/source/herd/asynctest/match/iterableMatchers.ceylon b/source/herd/asynctest/match/iterableMatchers.ceylon index 292fd8d..6cc3037 100644 --- a/source/herd/asynctest/match/iterableMatchers.ceylon +++ b/source/herd/asynctest/match/iterableMatchers.ceylon @@ -86,7 +86,7 @@ shared class DoesNotContain( "Item to check if matching stream doesn't co "Verifies if matching stream of `Sequence` does _not_ contain duplicate elements." -tagged( "Streams" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams" ) since( "0.7.0" ) by( "Lis" ) shared class ContainsNoDuplicates() satisfies Matcher<[Value*]> { diff --git a/source/herd/asynctest/match/listMatchers.ceylon b/source/herd/asynctest/match/listMatchers.ceylon index 1e2ece8..a9b4325 100644 --- a/source/herd/asynctest/match/listMatchers.ceylon +++ b/source/herd/asynctest/match/listMatchers.ceylon @@ -94,7 +94,7 @@ shared class Included ( "Verifies if matching `List` value is included in the given [[list]] at the given [[index]] of the [[list]]. `SearchableList.includesAt` is used in order to perform the verification. " -tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "List" ) since( "0.7.0" ) by( "Lis" ) shared class IncludedAt ( "The index at which the matching value might occur." Integer index, "List which is expected to include matching value." SearchableList list @@ -117,7 +117,7 @@ shared class IncludedAt ( "Verifies if matching value occurs in the given [[list]] at any index within [[from]]:[[length]] segment. `SearchableList.occurs` is used in order to perform the verification. " -tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "List" ) since( "0.7.0" ) by( "Lis" ) shared class Occured ( "List which is expected to include matching value." SearchableList list, "The smallet index to consider." Integer from = 0, @@ -141,7 +141,7 @@ shared class Occured ( "Verifies if matching value occurs in the given [[list]] at the given [[index]]. `SearchableList.occursAt` is used in order to perform the verification. " -tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "List" ) since( "0.7.0" ) by( "Lis" ) shared class OccuredAt ( "The index at which the matching value might occur." Integer index, "List which is expected to include matching value at the given [[index]]." SearchableList list @@ -184,7 +184,7 @@ shared class Includes ( "Verifies if matching `SearchableList` value includes the given [[list]] at the given [[index]]. `SearchableList.includesAt` is used in order to perform the verification. " -tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "List" ) since( "0.7.0" ) by( "Lis" ) shared class IncludesAt ( "The index at which [[list]] might occur in matching value." Integer index, "List which is expected to be included into matching one at the given [[index]]." List list @@ -207,7 +207,7 @@ shared class IncludesAt ( "Verifies if the given [[element]] occurs in matching `SearchableList` value within `from`:`length` segment. `SearchableList.occurs` is used in order to perform the verification. " -tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "List" ) since( "0.7.0" ) by( "Lis" ) shared class Occurs ( "Value which is expected to occur in the matching one at within `from`:`length` segment." Value element, "The smallet index to consider." Integer from = 0, @@ -232,7 +232,7 @@ shared class Occurs ( "Verifies if the given [[element]] occurs in matching `SearchableList` value at the given [[index]]. `SearchableList.occursAt` is used in order to perform the verification. " -tagged( "Streams", "List" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "List" ) since( "0.7.0" ) by( "Lis" ) shared class OccursAt ( "The index at which the given [[element]] might occur in the matching list." Integer index, "Value which is expected to occur in the matching one at within `from`:`length` segment." Value element diff --git a/source/herd/asynctest/match/mapMatchers.ceylon b/source/herd/asynctest/match/mapMatchers.ceylon index d99830c..ab1bd22 100644 --- a/source/herd/asynctest/match/mapMatchers.ceylon +++ b/source/herd/asynctest/match/mapMatchers.ceylon @@ -20,7 +20,7 @@ shared class DefinesKey( "Key to check if map defines." Value key ) "Verifies if matching map doesn't define the given key [[key]], see `Map.defines`." see( `class DoesNotContainItem`, `class ContainsItem`, `class DefinesKey` ) -tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "Maps" ) since( "0.7.0" ) by( "Lis" ) shared class DoesNotDefineKey( "Key to check if map doesn't define." Value key ) satisfies Matcher> given Value satisfies Object @@ -48,7 +48,7 @@ shared class ContainsItem( "Item to check if map contains." Value item ) "Verifies if matching map doesn't contain the given item [[item]]." see( `class ContainsItem`, `class DoesNotDefineKey`, `class DefinesKey` ) -tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "Maps" ) since( "0.7.0" ) by( "Lis" ) shared class DoesNotContainItem( "Item to check if map doesn't contain." Value item ) satisfies Matcher> given Value satisfies Object @@ -81,7 +81,7 @@ shared class ItemByKey ( "Verifies if the given [[map]] contains the given item [[item]] at the matching value as key. Items are compared using operator `==`." see( `class ItemByKey` ) -tagged( "Streams", "Maps" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Streams", "Maps" ) since( "0.7.0" ) by( "Lis" ) shared class ItemAtKey ( "Map which is expected to contain [[item]] under matching key." Map map, "Item to check if [[map]] contains under matching key." Value item diff --git a/source/herd/asynctest/match/numberMatchers.ceylon b/source/herd/asynctest/match/numberMatchers.ceylon index f48fee5..252df37 100644 --- a/source/herd/asynctest/match/numberMatchers.ceylon +++ b/source/herd/asynctest/match/numberMatchers.ceylon @@ -19,7 +19,7 @@ shared class IsNegative() "Verifies if matching value is _not_ negative, i.e. is >= 0." see( `class IsPositive`, `class IsNotPositive`, `class IsNegative`, `class IsZero`, `class IsNotZero` ) -tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Numbers" ) since( "0.7.0" ) by( "Lis" ) shared class IsNotNegative() satisfies Matcher given Value satisfies Number @@ -47,7 +47,7 @@ shared class IsPositive() "Verifies if matching value is not positive, i.e. is <= 0." see( `class IsNegative`, `class IsPositive`, `class IsNotNegative`, `class IsZero`, `class IsNotZero` ) -tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Numbers" ) since( "0.7.0" ) by( "Lis" ) shared class IsNotPositive() satisfies Matcher given Value satisfies Number @@ -140,7 +140,7 @@ shared class IsDefined() "Verifies if matching integer is even, i.e. if exists number that i=2*k." -tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Numbers" ) since( "0.7.0" ) by( "Lis" ) shared class IsEven() satisfies Matcher { shared actual MatchResult match( Integer val ) @@ -151,7 +151,7 @@ shared class IsEven() satisfies Matcher "Verifies if matching integer is odd, i.e. if exists number that i=2*k+1." -tagged( "Numbers" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Numbers" ) since( "0.7.0" ) by( "Lis" ) shared class IsOdd() satisfies Matcher { shared actual MatchResult match( Integer val ) diff --git a/source/herd/asynctest/match/setMatchers.ceylon b/source/herd/asynctest/match/setMatchers.ceylon index 452e33b..3aa8ffb 100644 --- a/source/herd/asynctest/match/setMatchers.ceylon +++ b/source/herd/asynctest/match/setMatchers.ceylon @@ -5,7 +5,7 @@ import herd.asynctest.internal { "Verifies if matching set is subset of the given [[superset]]." -tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Set" ) since( "0.7.0" ) by( "Lis" ) shared class SubsetOf ( "Set which is expected to be superset of matching set." Set superset ) @@ -22,7 +22,7 @@ shared class SubsetOf ( "Verifies if matching set is superset of the given [[subset]]." -tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Set" ) since( "0.7.0" ) by( "Lis" ) shared class SupersetOf ( "Set which is expected to be subset of matching set." Set subset ) @@ -39,7 +39,7 @@ shared class SupersetOf ( "Verifies if the given [[subset]] is subset of matching set." -tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Set" ) since( "0.7.0" ) by( "Lis" ) shared class Subset ( "Set which is expected to be subset of matching set." Set subset ) @@ -56,7 +56,7 @@ shared class Subset ( "Verifies if the given [[superset]] is superset of matching set." -tagged( "Set" ) since( "0.6.1" ) by( "Lis" ) +tagged( "Set" ) since( "0.7.0" ) by( "Lis" ) shared class Superset ( "Set which is expected to be superset of matching set." Set superset ) diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index 8370d9f..32db8e6 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -44,8 +44,9 @@ 8. [Matchers.](#matchers) 9. [Time out.](#timeout_section) 10. [Retry test.](#retry_section) - 11. [Conditional execution.](#conditions) - 12. [Reporting test results using charts.](#charts) + 11. [Conditional execution.](#conditions) + 12. [Benchmarking.](#benchmarking) + 13. [Reporting test results using charts.](#charts) ------------------------------------------- @@ -328,17 +329,23 @@ All conditions at every level are evaluated before test execution started and if some conditions are _not_ met (are unsuccessfull) the test is skipped and all rejection reasons are reported. - For an example, see, `ceylon.test::ignore` annotation. + For an example, see `ceylon.test::ignore` annotation. > Conditions are evaluation up to the first unsatisfied condition. So, there is no guarantee for a condition to be evaluated. + ------------------------------------------- + ### Benchmarking + + [[package herd.asynctest.benchmark]] package contains library to perform benchmark testing. + + ------------------------------------------- ### Reporting test results using charts Chart is simply a set of plots, where each plot is a sequence of 2D points. - Test results can be represented and reported with charts using stuff provided by [[package herd.asynctest.chart]] package. + Test results can be represented and reported with charts using [[package herd.asynctest.chart]] package. -------------------------------------------- @@ -367,9 +374,10 @@ license ( ) by( "Lis" ) native( "jvm" ) -module herd.asynctest "0.6.1" { +module herd.asynctest "0.7.0" { import java.base "8"; shared import ceylon.test "1.3.1"; import ceylon.collection "1.3.1"; shared import ceylon.file "1.3.1"; + import java.management "8"; } diff --git a/source/herd/asynctest/parameterization/ArgumentVariants.ceylon b/source/herd/asynctest/parameterization/ArgumentVariants.ceylon index 3f9ed43..23fe9eb 100644 --- a/source/herd/asynctest/parameterization/ArgumentVariants.ceylon +++ b/source/herd/asynctest/parameterization/ArgumentVariants.ceylon @@ -6,7 +6,7 @@ import ceylon.language.meta.declaration { "Specifies variants for a test function argument." tagged( "Combinatorial generators" ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared final class ArgumentVariants ( "Test function declaration." shared FunctionDeclaration testFunction, diff --git a/source/herd/asynctest/parameterization/CombinatorialKind.ceylon b/source/herd/asynctest/parameterization/CombinatorialKind.ceylon index a588e43..3edd08f 100644 --- a/source/herd/asynctest/parameterization/CombinatorialKind.ceylon +++ b/source/herd/asynctest/parameterization/CombinatorialKind.ceylon @@ -3,29 +3,29 @@ "Indicates the kind of the combinatorial source data." tagged( "Kind" ) see( `function combinatorial`, `class ArgumentVariants` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared abstract class CombinatorialKind() {} "Indicates that the test data has to be zipped." tagged( "Kind" ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared abstract class ZippedKind() of zippedKind extends CombinatorialKind() { string => "zipped combinatorial kind"; } "Indicates that the test data has to be zipped." tagged( "Kind" ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared object zippedKind extends ZippedKind() {} "Indicates that the test data has to be permuted." tagged( "Kind" ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared abstract class PermutationKind() of permutationKind extends CombinatorialKind() { string => "permutation combinatorial kind"; } "Indicates that the test data has to be permuted." tagged( "Kind" ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared object permutationKind extends PermutationKind() {} diff --git a/source/herd/asynctest/parameterization/MixingEnumerator.ceylon b/source/herd/asynctest/parameterization/MixingEnumerator.ceylon index 6ebc357..b663d7a 100644 --- a/source/herd/asynctest/parameterization/MixingEnumerator.ceylon +++ b/source/herd/asynctest/parameterization/MixingEnumerator.ceylon @@ -3,7 +3,7 @@ "Enumerates mixed variants." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) class MixingEnumerator( ArgumentVariants[] arguments ) satisfies TestVariantEnumerator { diff --git a/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon b/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon index 4480a80..e849b16 100644 --- a/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon +++ b/source/herd/asynctest/parameterization/TestVariantEnumerator.ceylon @@ -49,7 +49,7 @@ class TestVariantIterator ( "Delegates enumeration to another enumerator but completes when number of failures reach limit." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) class TestVariantMaxFailureEnumerator ( "Test variants iterator - function which returns next." TestVariantEnumerator other, "Limit on failures. When it is reached `next` returns `finished`." Integer maxFailedVariants diff --git a/source/herd/asynctest/parameterization/combinatorial.ceylon b/source/herd/asynctest/parameterization/combinatorial.ceylon index a20f736..485c483 100644 --- a/source/herd/asynctest/parameterization/combinatorial.ceylon +++ b/source/herd/asynctest/parameterization/combinatorial.ceylon @@ -21,7 +21,7 @@ import herd.asynctest.internal { tagged( "Applying combinatorial" ) see( `function combinatorial`, `class ParameterizedAnnotation`, `class DataSourceAnnotation`, `function permuting`, `function zipping`, `function mixing` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared final annotation class CombinatorialAnnotation ( "Test variant generator function which has to return [[TestVariantEnumerator]] and takes `ArgumentVariants[]` which is generated based on [[DataSourceAnnotation]] at each test function argument. @@ -84,7 +84,7 @@ shared final annotation class CombinatorialAnnotation ( See details in [[CombinatorialAnnotation]]." tagged( "Applying combinatorial" ) see( `function permuting`, `function zipping`, `function mixing` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation CombinatorialAnnotation combinatorial ( "Combinator function which has to return [[TestVariantEnumerator]] and takes `ArgumentVariants[]` which is generated based on [[DataSourceAnnotation]] at each test function argument. diff --git a/source/herd/asynctest/parameterization/dataSource.ceylon b/source/herd/asynctest/parameterization/dataSource.ceylon index a535257..146eec0 100644 --- a/source/herd/asynctest/parameterization/dataSource.ceylon +++ b/source/herd/asynctest/parameterization/dataSource.ceylon @@ -10,7 +10,7 @@ import ceylon.language.meta.declaration { The kind is used by _variant generator_ in order to identify strategy for the variant generations." tagged( "Data source" ) see( `class CombinatorialAnnotation` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared final annotation class DataSourceAnnotation ( "The source function or value declaration which has to return a stream of values. The source may be either top-level or tested class shared member." @@ -25,7 +25,7 @@ shared final annotation class DataSourceAnnotation ( "Provides values for the marked test function argument. See [[DataSourceAnnotation]] for details." tagged( "Data source" ) see( `function zippedSource`, `function permutationSource` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation DataSourceAnnotation dataSource ( "The source function or value declaration which has to return a stream of values. The source may be either top-level or tested class shared member." @@ -40,7 +40,7 @@ shared annotation DataSourceAnnotation dataSource ( The annotation is shortcut for `\`dataSource(`value zippedKind`)`\`." tagged( "Data source" ) see( `function dataSource`, `function permutationSource` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation DataSourceAnnotation zippedSource ( "The source function or value declaration which has to return a stream of values. The source may be either top-level or tested class shared member." @@ -53,7 +53,7 @@ shared annotation DataSourceAnnotation zippedSource ( The annotation is shortcut for `\`dataSource(`value permutationKind`)`\`." tagged( "Data source" ) see( `function zippedSource`, `function dataSource` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation DataSourceAnnotation permutationSource ( "The source function or value declaration which has to return a stream of values. The source may be either top-level or tested class shared member." diff --git a/source/herd/asynctest/parameterization/mixingCombinations.ceylon b/source/herd/asynctest/parameterization/mixingCombinations.ceylon index 0c590f0..6e9b279 100644 --- a/source/herd/asynctest/parameterization/mixingCombinations.ceylon +++ b/source/herd/asynctest/parameterization/mixingCombinations.ceylon @@ -32,7 +32,7 @@ tagged( "Applying combinatorial" ) see( `function combinatorial`, `function mixingCombinations`, `function permutationSource`, `function zippedSource`, `function permuting`, `function zipping` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation CombinatorialAnnotation mixing ( "Maximum number of failed variants to stop testing. Unlimited if <= 0." Integer maxFailedVariants = -1 @@ -40,7 +40,7 @@ shared annotation CombinatorialAnnotation mixing ( "Returns first permuted kind starting from index `from`." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) Integer firstPermutedKind( ArgumentVariants[] variants, Integer from ) { variable Integer retIndex = from; while ( exists arg = variants[retIndex ++], arg.kind === zippedKind ) {} @@ -57,7 +57,7 @@ Integer firstPermutedKind( ArgumentVariants[] variants, Integer from ) { " tagged( "Combinatorial generators" ) see( `function mixing`, `function permutationSource`, `function zippedSource` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared TestVariantEnumerator mixingCombinations ( "Variants of the test function arguments." ArgumentVariants[] arguments ) { diff --git a/source/herd/asynctest/parameterization/package.ceylon b/source/herd/asynctest/parameterization/package.ceylon index d283f45..5dd04a3 100644 --- a/source/herd/asynctest/parameterization/package.ceylon +++ b/source/herd/asynctest/parameterization/package.ceylon @@ -118,6 +118,28 @@ > Custom _variant generator_ has to take a list of [[ArgumentVariants]] and has to return [[TestVariantEnumerator]]. + #### Example + + Integer[] firstArgument => [1,2]; + Integer[] secondArgument => [10,20]; + String[] signArgument => [\"+\",\"-\"]; + + shared test mixing void testMixing ( + AsyncTestContext context, + zippedSource(\`value firstArgument\`) Integer arg1, + permutationSource(\`value signArgument\`) String arg2, + zippedSource(\`value secondArgument\`) Integer arg3 + ) { + context.succeed( \"\`\`arg1\`\`\`\`arg2\`\`\`\`arg3\`\`\" ); + context.complete(); + } + + In the above example `testMixing` is called 4 times with following arguments: + 1. 1, +, 10 + 2. 1, -, 10 + 3. 2, +, 20 + 4. 2, -, 20 + " -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared package herd.asynctest.parameterization; diff --git a/source/herd/asynctest/parameterization/permutingCombinations.ceylon b/source/herd/asynctest/parameterization/permutingCombinations.ceylon index 3aa7450..991265a 100644 --- a/source/herd/asynctest/parameterization/permutingCombinations.ceylon +++ b/source/herd/asynctest/parameterization/permutingCombinations.ceylon @@ -24,7 +24,7 @@ tagged( "Applying combinatorial" ) see( `function combinatorial`, `function permutingCombinations`, `function permutationSource`, `function zipping`, `function mixing` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation CombinatorialAnnotation permuting ( "Maximum number of failed variants to stop testing. Unlimited if <= 0." Integer maxFailedVariants = -1 @@ -32,7 +32,7 @@ shared annotation CombinatorialAnnotation permuting ( "Extracts argument by asserting existing." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) Anything getArgument( ArgumentVariants? args, Integer? index ) { assert ( exists args ); assert ( exists index ); @@ -49,7 +49,7 @@ Anything getArgument( ArgumentVariants? args, Integer? index ) { See example in [[permuting]]." tagged( "Combinatorial generators" ) see( `function permuting`, `function permutationSource` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared TestVariantEnumerator permutingCombinations ( "Variants of the test function arguments." ArgumentVariants[] arguments ) { diff --git a/source/herd/asynctest/parameterization/zippingCombinations.ceylon b/source/herd/asynctest/parameterization/zippingCombinations.ceylon index e8045e3..043f393 100644 --- a/source/herd/asynctest/parameterization/zippingCombinations.ceylon +++ b/source/herd/asynctest/parameterization/zippingCombinations.ceylon @@ -22,7 +22,7 @@ tagged( "Applying combinatorial" ) see( `function combinatorial`, `function zippingCombinations`, `function zippedSource`, `function permuting`, `function mixing` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation CombinatorialAnnotation zipping ( "Maximum number of failed variants to stop testing. Unlimited if <= 0." Integer maxFailedVariants = -1 @@ -30,7 +30,7 @@ shared annotation CombinatorialAnnotation zipping ( "Indicates that iterator finished." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) abstract class FinishedIndicator() of finishedIndicator {} object finishedIndicator extends FinishedIndicator() {} @@ -49,7 +49,7 @@ object finishedIndicator extends FinishedIndicator() {} " tagged( "Combinatorial generators" ) see( `function zipping`, `function zippedSource` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared TestVariantEnumerator zippingCombinations ( "Variants of the test function arguments." ArgumentVariants[] arguments ) { diff --git a/source/herd/asynctest/rule/ChainedTestContext.ceylon b/source/herd/asynctest/rule/ChainedTestContext.ceylon index ff52fe8..78a9975 100644 --- a/source/herd/asynctest/rule/ChainedTestContext.ceylon +++ b/source/herd/asynctest/rule/ChainedTestContext.ceylon @@ -14,7 +14,7 @@ import herd.asynctest.internal { "Provides in chain calls of the given test functions." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) class ChainedTestContext ( "Context to delegate reports." AsyncTestContext context, "The functions to be executed." Iterator functions ) { diff --git a/source/herd/asynctest/rule/ContextualRule.ceylon b/source/herd/asynctest/rule/ContextualRule.ceylon index 5918b63..09b272f 100644 --- a/source/herd/asynctest/rule/ContextualRule.ceylon +++ b/source/herd/asynctest/rule/ContextualRule.ceylon @@ -37,7 +37,7 @@ shared class ContextualRule( "Initial value source." Element | Element( class Nil {shared new instance {}} Element&Object|Nil getElement( Element | Element() source ) - => if ( exists ret = if ( is Element() source ) then source() else source) + => if ( exists ret = if ( is Element() source ) then source() else source ) then ret else Nil.instance; diff --git a/source/herd/asynctest/rule/CurrentTestStore.ceylon b/source/herd/asynctest/rule/CurrentTestStore.ceylon index f2aca05..26492e3 100644 --- a/source/herd/asynctest/rule/CurrentTestStore.ceylon +++ b/source/herd/asynctest/rule/CurrentTestStore.ceylon @@ -24,7 +24,7 @@ import java.util.concurrent.atomic { > Owner here is the test for which initialization has been performed. > The stored value is reseted by extracted from source value each time the new test is started. " -tagged( "TestRule" ) since( "0.6.1" ) by( "Lis" ) +tagged( "TestRule" ) since( "0.7.0" ) by( "Lis" ) shared class CurrentTestStore( "Source of the stored value." Element | Element() source ) satisfies TestRule { diff --git a/source/herd/asynctest/rule/MeterRule.ceylon b/source/herd/asynctest/rule/MeterRule.ceylon index 1be21e9..da98e92 100644 --- a/source/herd/asynctest/rule/MeterRule.ceylon +++ b/source/herd/asynctest/rule/MeterRule.ceylon @@ -4,6 +4,12 @@ import herd.asynctest { import java.util.concurrent.atomic { AtomicLong } +import herd.asynctest.benchmark { + StatisticSummary +} +import herd.asynctest.internal { + StatisticCalculator +} "Lock-free and thread-safely collects statistic data on an execution time and on a rate (per second) @@ -28,7 +34,7 @@ import java.util.concurrent.atomic { " see( `class StatisticSummary` ) -by( "Lis" ) since( "0.6.0" ) +since( "0.6.0" ) by( "Lis" ) tagged( "TestRule" ) shared class MeterRule() satisfies TestRule { diff --git a/source/herd/asynctest/rule/RuleChain.ceylon b/source/herd/asynctest/rule/RuleChain.ceylon index 505535e..37b8bdc 100644 --- a/source/herd/asynctest/rule/RuleChain.ceylon +++ b/source/herd/asynctest/rule/RuleChain.ceylon @@ -57,7 +57,7 @@ shared class TestRuleChain ( "Applies test statements in the given order. > If submited rule is marked with [[testRule]] annotation it will be executed twice." -tagged( "TestStatement" ) since( "0.6.1" ) by( "Lis" ) +tagged( "TestStatement" ) since( "0.7.0" ) by( "Lis" ) see( `class SuiteRuleChain`, `class TestRuleChain` ) shared class TestStatementChain ( "A list of the statements to be applied in order they are provided." TestStatement* statements diff --git a/source/herd/asynctest/rule/SignalRule.ceylon b/source/herd/asynctest/rule/SignalRule.ceylon index c10d36a..7770b15 100644 --- a/source/herd/asynctest/rule/SignalRule.ceylon +++ b/source/herd/asynctest/rule/SignalRule.ceylon @@ -42,7 +42,7 @@ import java.util.concurrent.atomic { } } " -tagged( "TestRule" ) since( "0.6.1" ) by( "Lis" ) +tagged( "TestRule" ) since( "0.7.0" ) by( "Lis" ) shared class SignalRule() satisfies TestRule { class Box() { diff --git a/source/herd/asynctest/rule/StatisticRule.ceylon b/source/herd/asynctest/rule/StatisticRule.ceylon index fb43303..9eee463 100644 --- a/source/herd/asynctest/rule/StatisticRule.ceylon +++ b/source/herd/asynctest/rule/StatisticRule.ceylon @@ -1,13 +1,19 @@ import herd.asynctest { AsyncPrePostContext } +import herd.asynctest.benchmark { + StatisticSummary +} +import herd.asynctest.internal { + StatisticCalculator +} "Lock-free and thread-safely accumulates statistics data of some variate values. Doesn't collect values, just accumulates statistic data when sample added - see [[sample]] and [[samples]]. Statistic data is reseted before _each_ test. " see( `class StatisticSummary` ) -tagged( "TestRule" ) by( "Lis" ) since( "0.6.0" ) +tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class StatisticRule() satisfies TestRule { diff --git a/source/herd/asynctest/rule/exceptions.ceylon b/source/herd/asynctest/rule/exceptions.ceylon index 26605b4..8f45ea7 100644 --- a/source/herd/asynctest/rule/exceptions.ceylon +++ b/source/herd/asynctest/rule/exceptions.ceylon @@ -2,6 +2,6 @@ "Exception thrown when access to a value is requested from a test which doesn't own the value." see( `class CurrentTestStore` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared class TestAccessDenied( String message ) extends Exception( message ) {} diff --git a/source/herd/asynctest/rule/ruleAnnotation.ceylon b/source/herd/asynctest/rule/ruleAnnotation.ceylon index c8ab998..2274236 100644 --- a/source/herd/asynctest/rule/ruleAnnotation.ceylon +++ b/source/herd/asynctest/rule/ruleAnnotation.ceylon @@ -31,7 +31,7 @@ shared annotation TestRuleAnnotation testRule() => TestRuleAnnotation(); > If statement from the given list is marked with [[testRule]] annotation it will be executed twice! " see( `interface TestStatement` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared final annotation class ApplyStatementAnnotation ( "Statement declarations. Each item has to satisfy [[TestStatement]] interface." shared ValueDeclaration* statements @@ -47,7 +47,7 @@ shared final annotation class ApplyStatementAnnotation ( > If statement from the given list is marked with [[testRule]] annotation it will be executed twice! " see( `interface TestRule`, `interface TestStatement` ) -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared annotation ApplyStatementAnnotation applyStatement ( "Statement declarations. Each item has to satisfy [[TestStatement]] interface." ValueDeclaration* statements diff --git a/source/herd/asynctest/runner/AsyncRunnerContext.ceylon b/source/herd/asynctest/runner/AsyncRunnerContext.ceylon index e960c24..8ae3253 100644 --- a/source/herd/asynctest/runner/AsyncRunnerContext.ceylon +++ b/source/herd/asynctest/runner/AsyncRunnerContext.ceylon @@ -3,7 +3,7 @@ "Base interface to push test messages to. The interface is mainly used by test runners, see package [[package herd.asynctest.runner]] for details. Test function receives [[herd.asynctest::AsyncTestContext]]." -since( "0.6.1" ) by( "Lis" ) +since( "0.7.0" ) by( "Lis" ) shared interface AsyncRunnerContext { "Completes the testing. To be called by the test function when testing is completed. From 4bf270023f5f50db51ddc8730cbc2253e9d1d162 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 11 Jan 2017 21:13:31 +0300 Subject: [PATCH 52/72] refactor --- .../asynctest/benchmark/plusMinus.ceylon | 2 +- .../herd/asynctest/benchmark/BaseBench.ceylon | 123 +++++++----------- .../herd/asynctest/benchmark/Options.ceylon | 5 +- .../herd/asynctest/internal/SimpleStat.ceylon | 10 ++ 4 files changed, 60 insertions(+), 80 deletions(-) diff --git a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon index 93491f2..74bf3ec 100644 --- a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon +++ b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon @@ -29,7 +29,7 @@ shared test async void plusMinusBenchmark(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(LocalIterations(20000).or(LocalError(0.002))), + Options(LocalIterations(20000).or(LocalError(0.002)), LocalIterations(1000).or(LocalError(0.01))), [SingleBench("plus", plusBenchmarkFunction), SingleBench("minus", minusBenchmarkFunction)], [1, 1], [2, 3], [25, 34] diff --git a/source/herd/asynctest/benchmark/BaseBench.ceylon b/source/herd/asynctest/benchmark/BaseBench.ceylon index 99139c1..0431d2a 100644 --- a/source/herd/asynctest/benchmark/BaseBench.ceylon +++ b/source/herd/asynctest/benchmark/BaseBench.ceylon @@ -25,45 +25,15 @@ shared abstract class BaseBench ( shared actual default Integer hash => memoizedHash else ( memoizedHash = 7 * title.hash + 37 ); - "Executed _before warmup round_ started. By default runs garbage collector." - shared default void beforeWarmupRound( Options options ) { - options.gcStrategy.gc( Stage.beforeWarmupRound ); - } - - "Executed _after warmup round_ has been completed. By default do nothing." - shared default void afterWarmupRound( Options options ) { - options.gcStrategy.gc( Stage.afterWarmupRound ); - } - - "Executed _before each_ warmup iteration. By default do nothing." - shared default void beforeWarmupIteration( Options options ) { - options.gcStrategy.gc( Stage.beforeWarmupIteration ); - } - - "Executed _after each_ warmup iteration. By default do nothing." - shared default void afterWarmupIteration( Options options ) { - options.gcStrategy.gc( Stage.afterWarmupIteration ); - } - - "Executed _before all_ measurements started. By default runs garbage collector." - shared default void beforeMeasureRound( Options options ) { - options.gcStrategy.gc( Stage.beforeMeasureRound ); - } - - "Executed _after all_ measurements have been completed. By default do nothing." - shared default void afterMeasureRound( Options options ) { - options.gcStrategy.gc( Stage.afterMeasureRound ); - } - - "Executed _before each_ iteration. By default do nothing." - shared default void beforeMeasureIteration( Options options ) { - options.gcStrategy.gc( Stage.beforeMeasureIteration ); - } - - "Executed _after each_ iteration. By default do nothing." - shared default void afterMeasureIteration( Options options ) { - options.gcStrategy.gc( Stage.afterMeasureIteration ); + "Executed before / after rounds and iterations as identified by [[stage]]. + By default runs garbage collector if specified by strategy, see [[GCStrategy]]." + shared default void stageEvent ( + "Options the bench is executed with." Options options, + "Stage the event belongs to." Stage stage + ) { + options.gcStrategy.gc( stage ); } + "Invokes a one test iteration. May return results in order to avoid dead-code elimination. @@ -80,21 +50,21 @@ shared abstract class BaseBench ( } "Executes test using the following cycle: - * [[beforeWarmupRound]] + * [[stageEvent]] with [[Stage.beforeWarmupRound]] * Cycle while [[Options.warmupCriterion]] is not met: - * [[beforeWarmupIteration]] + * [[stageEvent]] with [[Stage.beforeWarmupIteration]] * [[runIteration]] * consume result returned by [[runIteration]] using [[pushToBlackHole]] - * [[afterWarmupIteration]] - * [[afterWarmupRound]] - * [[beforeMeasureRound]] + * [[stageEvent]] with [[Stage.afterWarmupIteration]] + * [[stageEvent]] with [[Stage.afterWarmupRound]] + * [[stageEvent]] with [[Stage.beforeMeasureRound]] * Cycle while [[Options.measureCriterion]] is not met: - * [[beforeMeasureIteration]] + * [[stageEvent]] with [[Stage.beforeMeasureIteration]] * [[runIteration]] * collect time statistic * consume result returned by [[runIteration]] using [[pushToBlackHole]] - * [[afterMeasureIteration]] - * [[afterMeasureRound]] + * [[stageEvent]] with [[Stage.afterMeasureIteration]] + * [[stageEvent]] with [[Stage.afterMeasureRound]] > Note: runs when GC wroks are excluded from statistic calculations. " @@ -108,36 +78,18 @@ shared abstract class BaseBench ( // factor to scale time delta from nanoseconds (measured in) to timeUnit Float timeFactor = TimeUnit.nanoseconds.factorToSeconds / options.timeUnit.factorToSeconds; - // warmup round - clock is also warmupped! - if ( exists warmupCriterion = options.warmupCriterion ) { - SimpleStat calculator = SimpleStat(); - beforeWarmupRound( options ); - while ( true ) { - beforeWarmupIteration( options ); - clock.start(); - Anything ret = runIteration( *parameter ); - Float delta = clock.measure() * timeFactor; - if ( delta > 0.0 ) { - calculator.sample( 1.0 / delta ); - if ( warmupCriterion.verify( delta, calculator.result, options.timeUnit ) ) { - afterWarmupIteration( options ); - break; - } - } - afterWarmupIteration( options ); - pushToBlackHole( ret ); - } - afterWarmupRound( options ); - } - - // measure iterations + variable CompletionCriterion? warmupCriterion = options.warmupCriterion; SimpleStat calculator = SimpleStat(); - beforeMeasureRound( options ); + if ( warmupCriterion exists ) { stageEvent( options, Stage.beforeWarmupRound ); } + else { stageEvent( options, Stage.beforeMeasureRound ); } + + // bench iterations while ( true ) { // number of GC starts before test run Integer numGCBefore = numberOfGCRuns(); // execute the test - beforeMeasureIteration( options ); + if ( warmupCriterion exists ) { stageEvent( options, Stage.beforeWarmupIteration ); } + else { stageEvent( options, Stage.beforeMeasureIteration ); } clock.start(); Anything ret = runIteration( *parameter ); Float delta = clock.measure() * timeFactor; @@ -145,21 +97,36 @@ shared abstract class BaseBench ( if ( delta > 0.0 ) { // number of GC starts after test run Integer numGCAfter = numberOfGCRuns(); - if ( numGCAfter == numGCBefore ) { + if ( numGCAfter == numGCBefore || !options.skipGCRuns ) { // add sample only if GC has not been started during the test calculator.sample( 1.0 / delta ); - if ( options.measureCriterion.verify( delta, calculator.result, options.timeUnit ) ) { - // round is completed - afterMeasureIteration( options ); - break; + if ( exists criterion = warmupCriterion ) { + if ( criterion.verify( delta, calculator.result, options.timeUnit ) ) { + // warmup round is completed + warmupCriterion = null; + calculator.reset(); + stageEvent( options, Stage.afterWarmupIteration ); + stageEvent( options, Stage.afterWarmupRound ); + stageEvent( options, Stage.beforeMeasureRound ); + continue; + } + } + else { + if ( options.measureCriterion.verify( delta, calculator.result, options.timeUnit ) ) { + // measure round is completed + stageEvent( options, Stage.afterMeasureIteration ); + stageEvent( options, Stage.afterMeasureRound ); + break; + } } } } // completing iteration - afterMeasureIteration( options ); + if ( warmupCriterion exists ) { stageEvent( options, Stage.afterWarmupIteration ); } + else { stageEvent( options, Stage.afterMeasureIteration ); } pushToBlackHole( ret ); } - afterMeasureRound( options ); + return calculator.result; } diff --git a/source/herd/asynctest/benchmark/Options.ceylon b/source/herd/asynctest/benchmark/Options.ceylon index 971fea5..b8083dd 100644 --- a/source/herd/asynctest/benchmark/Options.ceylon +++ b/source/herd/asynctest/benchmark/Options.ceylon @@ -16,6 +16,9 @@ shared class Options ( shared Clock clock = Clock.wall, "Identifies GC execution strategy, i.e. how often to run GC. By default GC is executed before warmup round and before iteraton round." - shared GCStrategy gcStrategy = GCStagedStrategy.beforeRounds + shared GCStrategy gcStrategy = GCStagedStrategy.beforeRounds, + "`true` if runs when GC has been started has to be skipped and `flase` otherwise. + By default is `true`." + shared Boolean skipGCRuns = true ) { } diff --git a/source/herd/asynctest/internal/SimpleStat.ceylon b/source/herd/asynctest/internal/SimpleStat.ceylon index c66b528..a1752f2 100644 --- a/source/herd/asynctest/internal/SimpleStat.ceylon +++ b/source/herd/asynctest/internal/SimpleStat.ceylon @@ -27,6 +27,16 @@ shared class SimpleStat() { "The number of the values that have been statisticaly treated." shared Integer size => sizeVal; + + "Resets accumulated statistic to initial state." + shared void reset() { + minVal = infinity; + maxVal = -infinity; + meanVal = 0.0; + m2Val = 0.0; + sizeVal = 0; + } + "Returns variance of the values that have been statisticaly treated. The variance is mean((x-mean(x))^2)." shared Float variance => if ( sizeVal > 1 ) then m2Val / ( sizeVal - 1 ) else 0.0; From 74a0b031f2716a6bb3167f95ba6bdbcb284906d6 Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 12 Jan 2017 12:10:25 +0300 Subject: [PATCH 53/72] bench examples --- .../examples/asynctest/benchmark/isValueOrFunction.ceylon | 2 +- .../asynctest/benchmark/isWrapperOrFunction.ceylon | 8 +++----- .../herd/examples/asynctest/benchmark/plusMinus.ceylon | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon index fb2ccfa..e4a081c 100644 --- a/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon +++ b/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon @@ -51,7 +51,7 @@ shared test async void isValueOrFunction(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options( TotalIterations( 10000 ), TotalIterations( 100 ) ), + Options(TotalIterations(10000), TotalIterations(2000)), [ SingleBench("value", isValue), SingleBench("function", isFunction) diff --git a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon index 2e6c006..a418718 100644 --- a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon +++ b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon @@ -12,9 +12,7 @@ import herd.asynctest.benchmark { TimeUnit, Options, Clock, - TotalIterations, - LocalIterations, - LocalError + TotalIterations } @@ -69,7 +67,7 @@ shared test async void isWrapperOrFunctionCPU(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options( LocalIterations( 10000 ).or( LocalError( 0.005 ) ), LocalIterations( 1000 ), TimeUnit.milliseconds, Clock.cpu ), + Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, Clock.cpu), [ EmptyParameterBench ( "narrowed single argument wrapper", @@ -113,7 +111,7 @@ shared test async void isWrapperOrFunctionWall(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options( TotalIterations( 10000 ), TotalIterations( 1000 ), TimeUnit.milliseconds, Clock.wall ), + Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, Clock.wall), [ EmptyParameterBench ( "narrowed single argument wrapper", diff --git a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon index 74bf3ec..fd2e214 100644 --- a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon +++ b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon @@ -29,7 +29,7 @@ shared test async void plusMinusBenchmark(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(LocalIterations(20000).or(LocalError(0.002)), LocalIterations(1000).or(LocalError(0.01))), + Options(LocalIterations(20000).or(LocalError(0.002)), LocalIterations(2000).or(LocalError(0.01))), [SingleBench("plus", plusBenchmarkFunction), SingleBench("minus", minusBenchmarkFunction)], [1, 1], [2, 3], [25, 34] From 8b3fac80d6d1061b3ca61f88ef92e1134844ec0f Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 12 Jan 2017 17:26:57 +0300 Subject: [PATCH 54/72] execution time criterion --- .../asynctest/benchmark/TimeCriteria.ceylon | 57 +++++++++++++++++-- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/source/herd/asynctest/benchmark/TimeCriteria.ceylon b/source/herd/asynctest/benchmark/TimeCriteria.ceylon index b207cdc..c0e8c09 100644 --- a/source/herd/asynctest/benchmark/TimeCriteria.ceylon +++ b/source/herd/asynctest/benchmark/TimeCriteria.ceylon @@ -9,14 +9,54 @@ import java.util.concurrent { } +"Continues benchmark iterations while total execution time doesn't exceed [[timeLimit]]. + Execution time is overall time spent on test execution. + I.e. it summarizes time spent on benchmark function execution as well as on internal calculations. + + The alternative way is take into account time spent on benchmark function execution only is realized + in [[LocalBenchTime]] and [[TotalBenchTime]]. + " +tagged( "Criteria" ) +see( `class Options`, `class LocalBenchTime`, `class TotalBenchTime` ) +throws( `class AssertionError`, "Maximum permited execution is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class TotalExecutionTime ( + "Maximum permited execution time in [[timeUnit]] units." Integer timeLimit, + "Unit the [[timeLimit]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds +) + satisfies CompletionCriterion +{ + "Maximum permited execution has to be > 0." + assert( timeLimit > 0 ); + Integer totalNanoseconds = ( timeLimit * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + + variable Integer startTime = 0; + + + shared actual void reset() { + startTime = system.nanoseconds; + } + + shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit deltaTimeUnit ) { + return system.nanoseconds - startTime > totalNanoseconds; + } + +} + + "Continues benchmark iterations while overall time accumulated by benchmark iterations doesn't exceed - [[totalTime]]." + [[totalTime]]. + This criterion summarizes only time spent on benchmark function execution for all execution threads jointly. + + If benchmark function execution time is to be summarized for each thread separately [[LocalBenchTime]] is to be used. + If overall execution time has to be taken into account [[TotalExecutionTime]] is to be used. + " tagged( "Criteria" ) -see( `class Options`, `class LocalBenchTime` ) +see( `class Options`, `class LocalBenchTime`, `class TotalExecutionTime` ) throws( `class AssertionError`, "Total benchmark time is <= 0." ) since( "0.7.0" ) by( "Lis" ) shared class TotalBenchTime ( - "Maximum time (in [[timeUnit]]) to be accumulated by benchmark iterations." Integer totalTime, + "Maximum time (in [[timeUnit]] units) to be accumulated by benchmark iterations." Integer totalTime, "Unit the [[totalTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds ) satisfies CompletionCriterion @@ -41,13 +81,18 @@ shared class TotalBenchTime ( "Continues benchmark iterations while local (relative to thread) time accumulated by benchmark iterations doesn't exceed - [[localTime]]." + [[localTime]]. + This criterion summarizes time spent on benchmark function execution for each thread separately. + + If benchmark function execution time is to be summarized for all threads [[TotalBenchTime]] is to be used. + If overall execution time has to be taken into account [[TotalExecutionTime]] is to be used. + " tagged( "Criteria" ) -see( `class Options`, `class TotalBenchTime` ) +see( `class Options`, `class TotalBenchTime`, `class TotalExecutionTime` ) throws( `class AssertionError`, "Local benchmark time is <= 0." ) since( "0.7.0" ) by( "Lis" ) shared class LocalBenchTime ( - "Maximum time (in [[timeUnit]]) to be accumulated by local (relative to thread) iterations." Integer localTime, + "Maximum time (in [[timeUnit]] units) to be accumulated by local (relative to thread) iterations." Integer localTime, "Unit the [[localTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds ) satisfies CompletionCriterion From 287b03f9abb40c26ef9537dd0cd1da9aa9ec5a52 Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 12 Jan 2017 19:03:37 +0300 Subject: [PATCH 55/72] refactor --- .../asynctest/benchmark/plusMinus.ceylon | 2 +- .../herd/asynctest/benchmark/BaseBench.ceylon | 21 ++++++++++--------- .../benchmark/EmptyParameterBench.ceylon | 2 +- .../asynctest/benchmark/SingleBench.ceylon | 2 +- .../asynctest/benchmark/contextWriters.ceylon | 18 ++++++++++------ 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon index fd2e214..ae41651 100644 --- a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon +++ b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon @@ -29,7 +29,7 @@ shared test async void plusMinusBenchmark(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(LocalIterations(20000).or(LocalError(0.002)), LocalIterations(2000).or(LocalError(0.01))), + Options(LocalIterations(50000).or(LocalError(0.001)), LocalIterations(5000).or(LocalError(0.001))), [SingleBench("plus", plusBenchmarkFunction), SingleBench("minus", minusBenchmarkFunction)], [1, 1], [2, 3], [25, 34] diff --git a/source/herd/asynctest/benchmark/BaseBench.ceylon b/source/herd/asynctest/benchmark/BaseBench.ceylon index 0431d2a..6dba973 100644 --- a/source/herd/asynctest/benchmark/BaseBench.ceylon +++ b/source/herd/asynctest/benchmark/BaseBench.ceylon @@ -8,7 +8,7 @@ import herd.asynctest.internal { "Intended to implement details of bench execution in a signle thread. - Just [[runIteration]] has to be implemented which is indended to invoke test function once. + Just [[bench]] has to be implemented which is indended to invoke test function once. See [[execute]] for the run cycle. " tagged( "Bench" ) @@ -35,10 +35,10 @@ shared abstract class BaseBench ( } - "Invokes a one test iteration. + "Bench test function. May return results in order to avoid dead-code elimination. Returned result is consumed using [[pushToBlackHole]]." - shared formal Anything(*Parameter) runIteration; + shared formal Anything(*Parameter) bench; "Returns number of GC runs up to now." @@ -53,16 +53,16 @@ shared abstract class BaseBench ( * [[stageEvent]] with [[Stage.beforeWarmupRound]] * Cycle while [[Options.warmupCriterion]] is not met: * [[stageEvent]] with [[Stage.beforeWarmupIteration]] - * [[runIteration]] - * consume result returned by [[runIteration]] using [[pushToBlackHole]] + * [[bench]] + * consume result returned by [[bench]] using [[pushToBlackHole]] * [[stageEvent]] with [[Stage.afterWarmupIteration]] * [[stageEvent]] with [[Stage.afterWarmupRound]] * [[stageEvent]] with [[Stage.beforeMeasureRound]] * Cycle while [[Options.measureCriterion]] is not met: * [[stageEvent]] with [[Stage.beforeMeasureIteration]] - * [[runIteration]] + * [[bench]] * collect time statistic - * consume result returned by [[runIteration]] using [[pushToBlackHole]] + * consume result returned by [[bench]] using [[pushToBlackHole]] * [[stageEvent]] with [[Stage.afterMeasureIteration]] * [[stageEvent]] with [[Stage.afterMeasureRound]] @@ -87,18 +87,19 @@ shared abstract class BaseBench ( while ( true ) { // number of GC starts before test run Integer numGCBefore = numberOfGCRuns(); - // execute the test + // before iteration event if ( warmupCriterion exists ) { stageEvent( options, Stage.beforeWarmupIteration ); } else { stageEvent( options, Stage.beforeMeasureIteration ); } + // execute the test function clock.start(); - Anything ret = runIteration( *parameter ); + Anything ret = bench( *parameter ); Float delta = clock.measure() * timeFactor; // calculate execution statistic if ( delta > 0.0 ) { // number of GC starts after test run Integer numGCAfter = numberOfGCRuns(); if ( numGCAfter == numGCBefore || !options.skipGCRuns ) { - // add sample only if GC has not been started during the test + // add sample only if GC has not been started during the test or GC runs have not be skipped calculator.sample( 1.0 / delta ); if ( exists criterion = warmupCriterion ) { if ( criterion.verify( delta, calculator.result, options.timeUnit ) ) { diff --git a/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon b/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon index 8f93c40..3051900 100644 --- a/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon +++ b/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon @@ -8,7 +8,7 @@ shared final class EmptyParameterBench ( "Bench title. Generally unique." String title, "Function to be tested." - shared actual Anything() runIteration + shared actual Anything() bench ) extends BaseBench<[]>( title ) { diff --git a/source/herd/asynctest/benchmark/SingleBench.ceylon b/source/herd/asynctest/benchmark/SingleBench.ceylon index 153fc4d..8695742 100644 --- a/source/herd/asynctest/benchmark/SingleBench.ceylon +++ b/source/herd/asynctest/benchmark/SingleBench.ceylon @@ -7,7 +7,7 @@ shared final class SingleBench ( "Bench title. Generally unique." String title, "Function to be tested." - shared actual Anything(*Parameter) runIteration + shared actual Anything(*Parameter) bench ) extends BaseBench( title ) given Parameter satisfies Anything[] diff --git a/source/herd/asynctest/benchmark/contextWriters.ceylon b/source/herd/asynctest/benchmark/contextWriters.ceylon index 89e6252..f6dcb85 100644 --- a/source/herd/asynctest/benchmark/contextWriters.ceylon +++ b/source/herd/asynctest/benchmark/contextWriters.ceylon @@ -31,7 +31,8 @@ shared void writeAbsolute( AsyncTestContext context, Result( AsyncTestContext context, Result( AsyncTestContext context, Result< + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to slowest; " - + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + + "runs = ``stat.size``" ); } } @@ -76,7 +79,8 @@ shared void writeRelativeToSlowest( AsyncTestContext context, Result< "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to slowest; " - + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + + "runs = ``stat.size``" ); } } @@ -102,7 +106,8 @@ shared void writeRelativeToFastest( AsyncTestContext context, Result< + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to fastest; " - + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + + "runs = ``stat.size``" ); } } @@ -112,7 +117,8 @@ shared void writeRelativeToFastest( AsyncTestContext context, Result< "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to fastest; " - + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%" + + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + + "runs = ``stat.size``" ); } } From 51d08d250ef2915a1a75e5619a38ec346c971533 Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 13 Jan 2017 15:28:16 +0300 Subject: [PATCH 56/72] clock - interface --- .../benchmark/isWrapperOrFunction.ceylon | 9 +- .../herd/asynctest/benchmark/BaseBench.ceylon | 5 +- source/herd/asynctest/benchmark/Clock.ceylon | 128 +++++++++--------- .../herd/asynctest/benchmark/Options.ceylon | 2 +- .../benchmark/ThreadLocalClock.ceylon | 36 +---- 5 files changed, 79 insertions(+), 101 deletions(-) diff --git a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon index a418718..5ffbd5d 100644 --- a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon +++ b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon @@ -11,8 +11,9 @@ import herd.asynctest.benchmark { EmptyParameterBench, TimeUnit, Options, - Clock, - TotalIterations + TotalIterations, + CPUClock, + WallClock } @@ -67,7 +68,7 @@ shared test async void isWrapperOrFunctionCPU(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, Clock.cpu), + Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, CPUClock()), [ EmptyParameterBench ( "narrowed single argument wrapper", @@ -111,7 +112,7 @@ shared test async void isWrapperOrFunctionWall(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, Clock.wall), + Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, WallClock()), [ EmptyParameterBench ( "narrowed single argument wrapper", diff --git a/source/herd/asynctest/benchmark/BaseBench.ceylon b/source/herd/asynctest/benchmark/BaseBench.ceylon index 6dba973..4ff572f 100644 --- a/source/herd/asynctest/benchmark/BaseBench.ceylon +++ b/source/herd/asynctest/benchmark/BaseBench.ceylon @@ -72,11 +72,10 @@ shared abstract class BaseBench ( Options options, Parameter parameter ) { - // intialize clock + // clockto measure time interval Clock clock = options.clock; - clock.initialize(); // factor to scale time delta from nanoseconds (measured in) to timeUnit - Float timeFactor = TimeUnit.nanoseconds.factorToSeconds / options.timeUnit.factorToSeconds; + Float timeFactor = clock.units.factorToSeconds / options.timeUnit.factorToSeconds; variable CompletionCriterion? warmupCriterion = options.warmupCriterion; SimpleStat calculator = SimpleStat(); diff --git a/source/herd/asynctest/benchmark/Clock.ceylon b/source/herd/asynctest/benchmark/Clock.ceylon index 2b30440..2553e77 100644 --- a/source/herd/asynctest/benchmark/Clock.ceylon +++ b/source/herd/asynctest/benchmark/Clock.ceylon @@ -1,6 +1,3 @@ -import java.lang.management { - ManagementFactory -} import java.lang { ThreadLocal } @@ -9,24 +6,15 @@ import java.util.\ifunction { } -"Measures the time interval with one of the three strategies: - - 1. [[wall]] time, i.e. the time provided by system wall clocks. - 2. [[cpu]] thread time which is time the thread occupies a CPU. - 3. Thread [[user]] time which is time the thread occupies a CPU in user mode, rather than in system mode. - - > Note: `cpu` and `user` times are evaluated approximately. - - +"Measures the time interval. + Usage: - 1. Call [[initialize]] before measure cycle. - 2. Call [[start]] before each measure. - 3. Call [[measure]] to measure time interval from the last [[start]] call. + 1. Call [[start]] before each measure. + 2. Call [[measure]] to measure time interval from the last [[start]] call. Example: - Clock clock = Clock.wall; - clock.initialize(); + Clock clock = WallClock(); for ( item in 0 : totalRuns ) { clock.start(); doOperation(); @@ -37,69 +25,85 @@ import java.util.\ifunction { So, if [[measure]] is called from _thread A_, then the interval is measured from the last call of [[start]] done on the same _thread A_. " -tagged( "Options" ) +tagged( "Options", "Clock" ) see( `class Options` ) since( "0.7.0" ) by( "Lis" ) -shared class Clock - of cpu | user | wall +shared interface Clock { - "Instantiates new local to the thread clock." - ThreadLocalClock() instantiate; + "Units the clock measures time interval in." + shared formal TimeUnit units; - shared actual String string; - + "Starts time interval measure from the current time and locally for the current thread." + shared formal void start(); - "Indicates that `ThreadMXBean.currentThreadCpuTime` has to be used for time measurement. - If CPU time is not supported uses `system.nanoseconds`." - shared new cpu { - // TODO: log clock selection? - if ( ManagementFactory.threadMXBean.threadCpuTimeSupported ) { - string = "CPU clock"; - instantiate = CPULocalClock; - } - else { - string = "wall clock"; - instantiate = WallLocalClock; - } - } + "Measures time interval from the last [[start]] call (in the same thread as `measure` called) and up to now." + shared formal Integer measure(); - "Indicates that `ThreadMXBean.currentThreadUserTime` has to be used for time measurement. - If CPU time is not supported uses `system.nanoseconds`." - shared new user { - // TODO: log clock selection? - if ( ManagementFactory.threadMXBean.threadCpuTimeSupported ) { - string = "CPU user clock"; - instantiate = UserLocalClock; - } - else { - string = "wall clock"; - instantiate = WallLocalClock; - } - } +} + + +"Wall clock - uses `system.nanoseconds` to measure time interval." +tagged( "Clock" ) +see( `class Options`, `class CPUClock`, `class UserClock` ) +since( "0.7.0" ) by( "Lis" ) +shared class WallClock() satisfies Clock +{ + ThreadLocal startWallTime = ThreadLocal(); - "Indicates that `system.nanoseconds` has to be used for time measurement." - shared new wall { - string = "wall clock"; - instantiate = WallLocalClock; + shared actual TimeUnit units => TimeUnit.nanoseconds; + + shared actual void start() { + startWallTime.set( system.nanoseconds ); } + shared actual Integer measure() => system.nanoseconds - startWallTime.get(); + shared actual String string => "wall clock"; +} + + +"CPU clock uses `ThreadMXBean.currentThreadCpuTime` to measure time interval." +tagged( "Clock" ) +see( `class Options`, `class UserClock`, `class WallClock` ) +since( "0.7.0" ) by( "Lis" ) +shared class CPUClock() satisfies Clock +{ + ThreadLocal localClock = ThreadLocal.withInitial ( object satisfies Supplier { - get() => instantiate(); + get() => CPULocalClock(); } ); + shared actual TimeUnit units => TimeUnit.nanoseconds; + + shared actual void start() => localClock.get().start(); + + shared actual Integer measure() => localClock.get().measure(); + + shared actual String string => "CPU clock"; +} + + +"User clock uses `ThreadMXBean.currentThreadUserTime` to measure time interval." +tagged( "Clock" ) +see( `class Options`, `class CPUClock`, `class WallClock` ) +since( "0.7.0" ) by( "Lis" ) +shared class UserClock() satisfies Clock +{ + + ThreadLocal localClock = ThreadLocal.withInitial ( + object satisfies Supplier { + get() => UserLocalClock(); + } + ); - "Initializes the clock locally for the current thread. - Has to be called before any measurements." - shared void initialize() => localClock.get().initialize(); + shared actual TimeUnit units => TimeUnit.nanoseconds; - "Starts time interval measure from the current time and locally for the current thread." - shared void start() => localClock.get().start(); + shared actual void start() => localClock.get().start(); - "Measures time interval from the last [[start]] call (in the same thread as `measure` called) and up to now." - shared Integer measure() => localClock.get().measure(); + shared actual Integer measure() => localClock.get().measure(); + shared actual String string => "User clock"; } diff --git a/source/herd/asynctest/benchmark/Options.ceylon b/source/herd/asynctest/benchmark/Options.ceylon index b8083dd..268a99e 100644 --- a/source/herd/asynctest/benchmark/Options.ceylon +++ b/source/herd/asynctest/benchmark/Options.ceylon @@ -13,7 +13,7 @@ shared class Options ( "Time unit the results have to be reported with. Default is seconds." shared TimeUnit timeUnit = TimeUnit.seconds, "Clock to measure time intervals. Default wall clock." - shared Clock clock = Clock.wall, + shared Clock clock = WallClock(), "Identifies GC execution strategy, i.e. how often to run GC. By default GC is executed before warmup round and before iteraton round." shared GCStrategy gcStrategy = GCStagedStrategy.beforeRounds, diff --git a/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon b/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon index f2ef961..1c149dd 100644 --- a/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon +++ b/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon @@ -7,9 +7,6 @@ import java.lang.management { since( "0.7.0" ) by( "Lis" ) interface ThreadLocalClock { - "Initializes the clock." - shared formal void initialize(); - "Starts time interval measure from the current time." shared formal void start(); @@ -18,22 +15,6 @@ interface ThreadLocalClock { } -"Wall clock uses `system.nanoseconds` to measure time interval." -since( "0.7.0" ) by( "Lis" ) -class WallLocalClock() satisfies ThreadLocalClock { - - variable Integer startWallTime = system.nanoseconds; - - shared actual void start() { - startWallTime = system.nanoseconds; - } - - shared actual void initialize() {} - - shared actual Integer measure() => system.nanoseconds - startWallTime; -} - - "Base for the CPU time interval measurements. Calculates a factor to scale wall time interval corresponds to measure request. The factor is moving averaged ratio of CPU time interval to wall time interval. @@ -48,26 +29,19 @@ abstract class ThreadClock() satisfies ThreadLocalClock { variable Integer lastWallTime = 0; variable Integer lastSample = 0; variable Float factor = 1.0; + + if ( !ManagementFactory.threadMXBean.threadCpuTimeEnabled ) { + ManagementFactory.threadMXBean.threadCpuTimeEnabled = true; + } "Returns current time based on strategy." shared formal Integer readCurrentTime(); - shared actual void initialize() { - value threadMXBean = ManagementFactory.threadMXBean; - if ( !threadMXBean.threadCpuTimeEnabled ) { - threadMXBean.threadCpuTimeEnabled = true; - } - lastSample = readCurrentTime(); - startWallTime = system.nanoseconds; - lastWallTime = startWallTime; - factor = 1.0; - } - shared actual void start() { Integer currentSample = readCurrentTime(); Integer currentWallTime = system.nanoseconds; - if ( currentSample > lastSample && currentWallTime > lastWallTime ) { + if ( lastSample > 0 && currentSample > lastSample && currentWallTime > lastWallTime ) { // Moving averaging factor! factor = 0.9 * factor + 0.1 * ( currentSample - lastSample ) / ( currentWallTime - lastWallTime ); lastSample = currentSample; From 773a3e4426fb1f51250577431f7707030c00d054 Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 15 Jan 2017 15:20:21 +0300 Subject: [PATCH 57/72] mutable stat calculation --- .../herd/asynctest/benchmark/BaseBench.ceylon | 45 +++++------ .../benchmark/CompletionCriterion.ceylon | 19 +++-- .../asynctest/benchmark/ErrorCriteria.ceylon | 23 ++++-- .../asynctest/benchmark/GCStrategy.ceylon | 77 +++++++------------ .../benchmark/IterationsCriteria.ceylon | 12 +-- .../herd/asynctest/benchmark/Options.ceylon | 21 ++++- source/herd/asynctest/benchmark/Stage.ceylon | 2 +- .../StatisticAggregator.ceylon} | 10 +-- .../asynctest/benchmark/TimeCriteria.ceylon | 6 +- .../herd/asynctest/benchmark/benchmark.ceylon | 62 ++++++++++----- .../internal/StatisticCalculator.ceylon | 6 +- .../internal/extractSourceValue.ceylon | 2 +- .../herd/asynctest/rule/StatisticRule.ceylon | 2 +- 13 files changed, 156 insertions(+), 131 deletions(-) rename source/herd/asynctest/{internal/SimpleStat.ceylon => benchmark/StatisticAggregator.ceylon} (93%) diff --git a/source/herd/asynctest/benchmark/BaseBench.ceylon b/source/herd/asynctest/benchmark/BaseBench.ceylon index 4ff572f..865f1b3 100644 --- a/source/herd/asynctest/benchmark/BaseBench.ceylon +++ b/source/herd/asynctest/benchmark/BaseBench.ceylon @@ -1,9 +1,6 @@ import java.lang.management { ManagementFactory } -import herd.asynctest.internal { - SimpleStat -} "Intended to implement details of bench execution in a signle thread. @@ -25,13 +22,13 @@ shared abstract class BaseBench ( shared actual default Integer hash => memoizedHash else ( memoizedHash = 7 * title.hash + 37 ); - "Executed before / after rounds and iterations as identified by [[stage]]. - By default runs garbage collector if specified by strategy, see [[GCStrategy]]." - shared default void stageEvent ( - "Options the bench is executed with." Options options, - "Stage the event belongs to." Stage stage + "Executes optioned callbacks." + void stageEvent ( + "Stage the event belongs to." Stage stage, "Callbacks." Anything(Stage)[] callbacks ) { - options.gcStrategy.gc( stage ); + for ( item in callbacks ) { + item( stage ); + } } @@ -72,23 +69,23 @@ shared abstract class BaseBench ( Options options, Parameter parameter ) { - // clockto measure time interval + // clock to measure time interval Clock clock = options.clock; // factor to scale time delta from nanoseconds (measured in) to timeUnit Float timeFactor = clock.units.factorToSeconds / options.timeUnit.factorToSeconds; variable CompletionCriterion? warmupCriterion = options.warmupCriterion; - SimpleStat calculator = SimpleStat(); - if ( warmupCriterion exists ) { stageEvent( options, Stage.beforeWarmupRound ); } - else { stageEvent( options, Stage.beforeMeasureRound ); } + StatisticAggregator calculator = StatisticAggregator(); + if ( warmupCriterion exists ) { stageEvent( Stage.beforeWarmupRound, options.callbacks ); } + else { stageEvent( Stage.beforeMeasureRound, options.callbacks ); } // bench iterations while ( true ) { // number of GC starts before test run Integer numGCBefore = numberOfGCRuns(); // before iteration event - if ( warmupCriterion exists ) { stageEvent( options, Stage.beforeWarmupIteration ); } - else { stageEvent( options, Stage.beforeMeasureIteration ); } + if ( warmupCriterion exists ) { stageEvent( Stage.beforeWarmupIteration, options.callbacks ); } + else { stageEvent( Stage.beforeMeasureIteration, options.callbacks ); } // execute the test function clock.start(); Anything ret = bench( *parameter ); @@ -101,29 +98,29 @@ shared abstract class BaseBench ( // add sample only if GC has not been started during the test or GC runs have not be skipped calculator.sample( 1.0 / delta ); if ( exists criterion = warmupCriterion ) { - if ( criterion.verify( delta, calculator.result, options.timeUnit ) ) { + if ( criterion.verify( delta, calculator, options.timeUnit ) ) { // warmup round is completed warmupCriterion = null; calculator.reset(); - stageEvent( options, Stage.afterWarmupIteration ); - stageEvent( options, Stage.afterWarmupRound ); - stageEvent( options, Stage.beforeMeasureRound ); + stageEvent( Stage.afterWarmupIteration, options.callbacks ); + stageEvent( Stage.afterWarmupRound, options.callbacks ); + stageEvent( Stage.beforeMeasureRound, options.callbacks ); continue; } } else { - if ( options.measureCriterion.verify( delta, calculator.result, options.timeUnit ) ) { + if ( options.measureCriterion.verify( delta, calculator, options.timeUnit ) ) { // measure round is completed - stageEvent( options, Stage.afterMeasureIteration ); - stageEvent( options, Stage.afterMeasureRound ); + stageEvent( Stage.afterMeasureIteration, options.callbacks ); + stageEvent( Stage.afterMeasureRound, options.callbacks ); break; } } } } // completing iteration - if ( warmupCriterion exists ) { stageEvent( options, Stage.afterWarmupIteration ); } - else { stageEvent( options, Stage.afterMeasureIteration ); } + if ( warmupCriterion exists ) { stageEvent( Stage.afterWarmupIteration, options.callbacks ); } + else { stageEvent( Stage.afterMeasureIteration, options.callbacks ); } pushToBlackHole( ret ); } diff --git a/source/herd/asynctest/benchmark/CompletionCriterion.ceylon b/source/herd/asynctest/benchmark/CompletionCriterion.ceylon index 1c0af8f..f45f80d 100644 --- a/source/herd/asynctest/benchmark/CompletionCriterion.ceylon +++ b/source/herd/asynctest/benchmark/CompletionCriterion.ceylon @@ -11,9 +11,14 @@ shared interface CompletionCriterion { "Returns `true` if completion criterion is met and `false` otherwise." shared formal Boolean verify ( - "The latest execution time in the given time unit." Float delta, - "Accumulated execution statistic of operations per given time unit." StatisticSummary result, - "Time unit [[delta]] and [[result]] are measured in." TimeUnit timeUnit + "The latest execution time in the given time unit." + Float delta, + "Accumulated execution statistic of operations per given time unit. + Statistic is aggregated by theexecution on the current thread. + This is mutable but be careful to modify aggregated statistic. It may lead to wrong summary results." + StatisticAggregator stat, + "Time unit [[delta]] and [[stat]] are measured in." + TimeUnit timeUnit ); "Combines `this` and `other` criterion using logical `and`. @@ -41,9 +46,9 @@ class AndCriterion( CompletionCriterion+ criteria ) satisfies CompletionCriterio } } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { for ( item in criteria ) { - if ( !item.verify( delta, result, timeUnit ) ) { + if ( !item.verify( delta, stat, timeUnit ) ) { return false; } } @@ -67,9 +72,9 @@ class OrCriterion( CompletionCriterion+ criteria ) satisfies CompletionCriterion } } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { for ( item in criteria ) { - if ( item.verify( delta, result, timeUnit ) ) { + if ( item.verify( delta, stat, timeUnit ) ) { return true; } } diff --git a/source/herd/asynctest/benchmark/ErrorCriteria.ceylon b/source/herd/asynctest/benchmark/ErrorCriteria.ceylon index c2f331c..6a62b64 100644 --- a/source/herd/asynctest/benchmark/ErrorCriteria.ceylon +++ b/source/herd/asynctest/benchmark/ErrorCriteria.ceylon @@ -23,16 +23,29 @@ shared class TotalError ( "Maximum allowed total error has to be > 0." assert( maxRelativeError > 0.0 ); - ConcurrentSkipListMap localStat = ConcurrentSkipListMap( IntegerComparator() ); + class Stat ( + shared variable Float mean, + shared variable Float variance, + shared variable Integer size + ) {} + + ConcurrentSkipListMap localStat = ConcurrentSkipListMap( IntegerComparator() ); shared actual void reset() { localStat.clear(); } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { Integer id = Thread.currentThread().id; - localStat.put( id, result ); + if ( exists localStat = localStat.get( id ) ) { + localStat.mean = stat.mean; + localStat.variance = stat.variance; + localStat.size = stat.size; + } + else { + localStat.put( id, Stat( stat.mean, stat.variance, stat.size ) ); + } variable Integer size = 0; variable Float mean = 0.0; variable Float variance = 0.0; @@ -72,8 +85,8 @@ shared class LocalError ( shared actual void reset() { } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { - return result.size > minIterations && result.relativeSampleError < maxRelativeError; + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { + return stat.size > minIterations && stat.relativeSampleError < maxRelativeError; } } diff --git a/source/herd/asynctest/benchmark/GCStrategy.ceylon b/source/herd/asynctest/benchmark/GCStrategy.ceylon index 7287e2f..c344a3a 100644 --- a/source/herd/asynctest/benchmark/GCStrategy.ceylon +++ b/source/herd/asynctest/benchmark/GCStrategy.ceylon @@ -3,61 +3,36 @@ import java.lang { } -"Strategy to run garbage collector." -tagged( "Options" ) +"Executes garbage collector at the given `stage`." +tagged( "Callbacks" ) see( `class Options` ) since( "0.7.0" ) by( "Lis" ) -shared interface GCStrategy { - "Runs garbage collector if all conditions meet the strategy." - shared formal void gc( "Current execution stage." Stage stage ); +shared void gcCallback( "Stages GC has to be executed at." Stage+ stages )( "Current stage." Stage stage ) { + if ( stage in stages ) { + System.gc(); + } } +"Executes garbage collector before warmup and measure rounds. " +tagged( "Callbacks" ) +see( `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared Anything(Stage) gcBeforeRounds = gcCallback( Stage.beforeWarmupRound, Stage.beforeMeasureRound ); -"Identifies at which execution stages garbage collector has to be run." -tagged( "Options" ) +"Executes garbage collector before measure round." +tagged( "Callbacks" ) see( `class Options` ) since( "0.7.0" ) by( "Lis" ) -shared final class GCStagedStrategy satisfies GCStrategy { - - Stage[] stageSet; - - - "Runs GC at a number of stages." - shared new combined( Stage+ stages ) { - stageSet = stages; - } - - "GC is to be never run." - shared new never { - stageSet = empty; - } - - "Runs GC before warmup round only." - shared new beforeWarmupRound { - stageSet = [Stage.beforeWarmupRound]; - } - - "Runs GC before measure round only." - shared new beforeMeasureRound { - stageSet = [Stage.beforeMeasureRound]; - } - - "Runs GC before warmup and measure rounds." - shared new beforeRounds { - stageSet = [Stage.beforeWarmupRound, Stage.beforeMeasureRound]; - } - - "Runs GC before each iteration." - shared new beforeEachIteration { - stageSet = [Stage.beforeWarmupIteration, Stage.beforeMeasureIteration]; - } - - - "Runs GC if `stage` satisfies the strategy requirements." - shared actual void gc( "Current execution stage." Stage stage ) { - if ( stageSet.contains( stage ) ) { - System.gc(); - } - } - -} +shared Anything(Stage) gcBeforeMeasureRound = gcCallback( Stage.beforeMeasureRound ); + +"Executes garbage collector before each measure iteration." +tagged( "Callbacks" ) +see( `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared Anything(Stage) gcBeforeMeasureIteration = gcCallback( Stage.beforeMeasureIteration ); + +"Executes garbage collector before each warmup and measure iteration." +tagged( "Callbacks" ) +see( `class Options` ) +since( "0.7.0" ) by( "Lis" ) +shared Anything(Stage) gcBeforeIterations = gcCallback( Stage.beforeWarmupIteration, Stage.beforeMeasureIteration ); diff --git a/source/herd/asynctest/benchmark/IterationsCriteria.ceylon b/source/herd/asynctest/benchmark/IterationsCriteria.ceylon index fffe4f0..d3e9c44 100644 --- a/source/herd/asynctest/benchmark/IterationsCriteria.ceylon +++ b/source/herd/asynctest/benchmark/IterationsCriteria.ceylon @@ -26,7 +26,7 @@ shared class TotalIterations ( counter.set( 1 ); } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { return counter.incrementAndGet() > numberOfIterations; } @@ -37,23 +37,23 @@ shared class TotalIterations ( Number of iterations is equal to number of calls of [[verify]] after last call of [[reset]]." tagged( "Criteria" ) see( `class Options`, `class TotalIterations` ) -throws( `class AssertionError`, "Thread local number of benchmark iterations is <= 0." ) +throws( `class AssertionError`, "Thread local restriction on a number of benchmark iterations is <= 0." ) since( "0.7.0" ) by( "Lis" ) shared class LocalIterations ( - "Thread local number of benchmark iterations. Has to be > 0." Integer numberOfIterations + "Thread local restriction on a number of benchmark iterations. Has to be > 0." Integer numberOfIterations ) satisfies CompletionCriterion { - "Thread local number of benchmark iterations has to be > 0." + "Thread local restriction on a number of benchmark iterations has to be > 0." assert( numberOfIterations > 0 ); shared actual void reset() { } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit timeUnit ) { - return result.size >= numberOfIterations; + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { + return stat.size >= numberOfIterations; } } diff --git a/source/herd/asynctest/benchmark/Options.ceylon b/source/herd/asynctest/benchmark/Options.ceylon index 268a99e..2fd5196 100644 --- a/source/herd/asynctest/benchmark/Options.ceylon +++ b/source/herd/asynctest/benchmark/Options.ceylon @@ -14,11 +14,24 @@ shared class Options ( shared TimeUnit timeUnit = TimeUnit.seconds, "Clock to measure time intervals. Default wall clock." shared Clock clock = WallClock(), - "Identifies GC execution strategy, i.e. how often to run GC. - By default GC is executed before warmup round and before iteraton round." - shared GCStrategy gcStrategy = GCStagedStrategy.beforeRounds, "`true` if runs when GC has been started has to be skipped and `flase` otherwise. By default is `true`." - shared Boolean skipGCRuns = true + shared Boolean skipGCRuns = true, + "Callbacks executed at each stage. + By default callback which executes garbage collector before warmup and measure rounds (see [[gcBeforeRounds]])." + shared Anything(Stage)[] callbacks = [gcBeforeRounds] ) { } + + +"Returns options with prepended callbacks." +since( "0.7.0" ) by( "Lis" ) +Options prependCallbacksToOptions( Options options, Anything(Stage)+ callbacks ) + => Options( options.measureCriterion, options.warmupCriterion, options.timeUnit, options.clock, options.skipGCRuns, + options.callbacks.prepend( callbacks ) ); + +"Returns options with appended callbacks." +since( "0.7.0" ) by( "Lis" ) +Options appendCallbacksToOptions( Options options, Anything(Stage)+ callbacks ) + => Options( options.measureCriterion, options.warmupCriterion, options.timeUnit, options.clock, options.skipGCRuns, + options.callbacks.append( callbacks ) ); diff --git a/source/herd/asynctest/benchmark/Stage.ceylon b/source/herd/asynctest/benchmark/Stage.ceylon index c8e00a3..cf19d76 100644 --- a/source/herd/asynctest/benchmark/Stage.ceylon +++ b/source/herd/asynctest/benchmark/Stage.ceylon @@ -1,7 +1,7 @@ "Identifies a stage of the bench execution." -see( `class Options`, `interface GCStrategy`, `class GCStagedStrategy` ) +see( `class Options`, `function gcCallback` ) tagged( "Options" ) since( "0.7.0" ) by( "Lis" ) shared class Stage diff --git a/source/herd/asynctest/internal/SimpleStat.ceylon b/source/herd/asynctest/benchmark/StatisticAggregator.ceylon similarity index 93% rename from source/herd/asynctest/internal/SimpleStat.ceylon rename to source/herd/asynctest/benchmark/StatisticAggregator.ceylon index a1752f2..c4e7b1c 100644 --- a/source/herd/asynctest/internal/SimpleStat.ceylon +++ b/source/herd/asynctest/benchmark/StatisticAggregator.ceylon @@ -1,14 +1,11 @@ import java.lang { Math } -import herd.asynctest.benchmark { - StatisticSummary -} "Provides thread-unsafe statistic calculations." since( "0.7.0" ) by( "Lis" ) -shared class SimpleStat() { +shared class StatisticAggregator() { variable Float minVal = infinity; variable Float maxVal = -infinity; @@ -28,7 +25,7 @@ shared class SimpleStat() { shared Integer size => sizeVal; - "Resets accumulated statistic to initial state." + "Resets accumulated statistic to zero state." shared void reset() { minVal = infinity; maxVal = -infinity; @@ -60,7 +57,7 @@ shared class SimpleStat() { "Equals to sampleError/mean." shared Float relativeSampleError => sampleError / mean; - + "Adds sample to the statistic data." shared void sample( Float val ) { minVal = val < minVal then val else minVal; maxVal = val > maxVal then val else maxVal; @@ -70,6 +67,7 @@ shared class SimpleStat() { m2Val = sizeVal > 1 then m2Val + delta * ( val - meanVal ) else 0.0; } + "Statistic summary aggregated up to the moment." shared StatisticSummary result => StatisticSummary( minVal, maxVal, meanVal, standardDeviation, sizeVal ); } diff --git a/source/herd/asynctest/benchmark/TimeCriteria.ceylon b/source/herd/asynctest/benchmark/TimeCriteria.ceylon index c0e8c09..fde3c3e 100644 --- a/source/herd/asynctest/benchmark/TimeCriteria.ceylon +++ b/source/herd/asynctest/benchmark/TimeCriteria.ceylon @@ -37,7 +37,7 @@ shared class TotalExecutionTime ( startTime = system.nanoseconds; } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit deltaTimeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit deltaTimeUnit ) { return system.nanoseconds - startTime > totalNanoseconds; } @@ -72,7 +72,7 @@ shared class TotalBenchTime ( accumulatedTime.set( 0 ); } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit deltaTimeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit deltaTimeUnit ) { Integer timeToAdd = ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; return accumulatedTime.addAndGet( timeToAdd ) > totalNanoseconds; } @@ -109,7 +109,7 @@ shared class LocalBenchTime ( localAccumulators.clear(); } - shared actual Boolean verify( Float delta, StatisticSummary result, TimeUnit deltaTimeUnit ) { + shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit deltaTimeUnit ) { Integer id = Thread.currentThread().id; Integer timeVal; if ( exists current = localAccumulators.get( id ) ) { diff --git a/source/herd/asynctest/benchmark/benchmark.ceylon b/source/herd/asynctest/benchmark/benchmark.ceylon index e7ede41..ef8f18e 100644 --- a/source/herd/asynctest/benchmark/benchmark.ceylon +++ b/source/herd/asynctest/benchmark/benchmark.ceylon @@ -1,6 +1,31 @@ import ceylon.collection { HashMap } +import java.lang { + Thread, + System +} + + +void initializeBench( Options options ) { + blackHole.clear(); + options.measureCriterion.reset(); + options.warmupCriterion?.reset(); +} + +void completeBench( Options options ) { + options.measureCriterion.reset(); + options.warmupCriterion?.reset(); + // avoid dead code elimination for the black hole + blackHole.verifyNumbers(); +} + +void warmupBenchmark() { + System.gc(); + Thread.sleep( 200 ); + System.gc(); + Thread.sleep( 200 ); +} "Runs benchmark testing. @@ -12,62 +37,61 @@ shared Result benchmark ( "Benchmark options of benches executing." Options options, "A list of benches to be executed." - Bench[] benches, + [Bench+] benches, "A list of parameters the test has to be executed with." Parameter* parameters ) given Parameter satisfies Anything[] -{ - if ( parameters nonempty ) { +{ + warmupBenchmark(); + + if ( nonempty params = parameters ) { + // run test for each parameter and each bench HashMap> res = HashMap>(); - for ( param in parameters ) { + for ( param in params ) { if ( !res.defines( param ) ) { HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary>(); for ( bench in benches ) { if ( !benchRes.defines( bench ) ) { - // initialize black hole - blackHole.clear(); - // initialize criteria - options.measureCriterion.reset(); - options.warmupCriterion?.reset(); + // initialize criteria and black hole + initializeBench( options ); // execute bench value br = bench.execute( options, param ); // store results benchRes.put( bench, br ); - // avoid dead code elimination for the black hole - blackHole.verifyNumbers(); + // reset criteria and verify black hole + completeBench( options ); } } res.put( param, ParameterResult( param, benchRes ) ); } } + completeBench( options ); return Result( options.timeUnit, res ); } else { // parameter list is empty! only empty parameters test is admited "benchmarking: if parameter list is empty, `Parameter` has to be `empty`." assert ( [] is Parameter ); - assert ( is Bench<[]>[] benches ); + assert ( is [Bench<[]>+] benches ); HashMap<[], ParameterResult<[]>> res = HashMap<[], ParameterResult<[]>>(); HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary>(); for ( bench in benches ) { if ( !benchRes.defines( bench ) ) { - // initialize black hole - blackHole.clear(); - // initialize criteria - options.measureCriterion.reset(); - options.warmupCriterion?.reset(); + // initialize criteria and black hole + initializeBench( options ); // execute bench value br = bench.execute( options, [] ); // store results benchRes.put( bench, br ); - // avoid dead code elimination for the black hole - blackHole.verifyNumbers(); + // reset criteria and verify black hole + completeBench( options ); } } res.put( [], ParameterResult<[]>( [], benchRes ) ); + completeBench( options ); assert ( is Result ret = Result<[]>( options.timeUnit, res ) ); return ret; } diff --git a/source/herd/asynctest/internal/StatisticCalculator.ceylon b/source/herd/asynctest/internal/StatisticCalculator.ceylon index 3d0c868..d516afc 100644 --- a/source/herd/asynctest/internal/StatisticCalculator.ceylon +++ b/source/herd/asynctest/internal/StatisticCalculator.ceylon @@ -34,7 +34,7 @@ class StatisticStream { } "New stream by the given values." - shared new withValues( Float* values ) { + shared new withValues( {Float*} values ) { variable Float min = infinity; variable Float max = -infinity; variable Float mean = 0.0; @@ -145,13 +145,13 @@ shared class StatisticCalculator() { "Thread-safely adds samples to the statistic." see( `value statisticSummary`, `function sample` ) - shared void samples( Float* values ) { + shared void samples( {Float*} values ) { Integer addedSize = values.size; if ( addedSize == 1, exists val = values.first ) { sample( val ); } else if ( addedSize > 1 ) { - StatisticStream sAdd = StatisticStream.withValues( *values ); + StatisticStream sAdd = StatisticStream.withValues( values ); variable StatisticStream sOld = stat.get(); variable StatisticStream sNew = StatisticStream.combined( sOld, sAdd ); while ( !stat.compareAndSet( sOld, sNew ) ) { diff --git a/source/herd/asynctest/internal/extractSourceValue.ceylon b/source/herd/asynctest/internal/extractSourceValue.ceylon index a7b187c..59dcfa0 100644 --- a/source/herd/asynctest/internal/extractSourceValue.ceylon +++ b/source/herd/asynctest/internal/extractSourceValue.ceylon @@ -22,5 +22,5 @@ shared Result extractSourceValue( FunctionOrValueDeclaration source, Obj return if ( !source.toplevel, exists instance ) then source.memberApply( type( instance ) ).bind( instance ).get() else source.apply().get(); - } + } } diff --git a/source/herd/asynctest/rule/StatisticRule.ceylon b/source/herd/asynctest/rule/StatisticRule.ceylon index 9eee463..2dfcf8c 100644 --- a/source/herd/asynctest/rule/StatisticRule.ceylon +++ b/source/herd/asynctest/rule/StatisticRule.ceylon @@ -31,7 +31,7 @@ shared class StatisticRule() satisfies TestRule "Thread-safely adds samples to the statistic." see( `value statisticSummary`, `function sample` ) - shared void samples( Float* values ) => calculator.element.samples( *values ); + shared void samples( {Float*} values ) => calculator.element.samples( values ); shared actual void after( AsyncPrePostContext context ) => calculator.after( context ); From e261a9d4b230ae94e48cdb2ba1cccd5104204e02 Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 15 Jan 2017 15:20:51 +0300 Subject: [PATCH 58/72] multithread bench parameterized with number of threads --- .../asynctest/benchmark/MultiBench.ceylon | 105 ++++++++++++++++++ .../herd/asynctest/benchmark/SyncBench.ceylon | 27 +++++ 2 files changed, 132 insertions(+) create mode 100644 source/herd/asynctest/benchmark/MultiBench.ceylon create mode 100644 source/herd/asynctest/benchmark/SyncBench.ceylon diff --git a/source/herd/asynctest/benchmark/MultiBench.ceylon b/source/herd/asynctest/benchmark/MultiBench.ceylon new file mode 100644 index 0000000..a426acd --- /dev/null +++ b/source/herd/asynctest/benchmark/MultiBench.ceylon @@ -0,0 +1,105 @@ +import java.util.concurrent.locks { + ReentrantLock +} +import java.util.concurrent { + CyclicBarrier, + CountDownLatch +} +import java.lang { + Thread, + Math +} + + +"Executes bench functions `benches` in separated threads. + Number of threads for each bench function is specified by the bench parameter with corresponding index. + I.e. given list of bench function `benches` and list of `Integer` (`parameter` argument of [[execute]]). + For each bench function with index `index` number of threads to execute the function is specified via + item of `parameter` list with the same index `index`. + So the bench is parameterized by number of execution threads. + " +tagged( "Bench" ) +since( "0.7.0" ) by( "Lis" ) +shared class MultiBench ( + shared actual String title, + "Bench functions." Anything()+ benches +) + satisfies Bench<[Integer+]> +{ + + ReentrantLock locker = ReentrantLock(); + variable Float min = infinity; + variable Float max = -infinity; + variable Float mean = 0.0; + variable Float variance = 0.0; + variable Integer size = 0; + + + void writeResults( CountDownLatch completeLatch )( StatisticSummary stat ) { + locker.lock(); + try { + if ( stat.min < min ) { + min = stat.min; + } + if ( stat.max > max ) { + max = stat.max; + } + mean += stat.mean; + variance += stat.variance; + size += stat.size; + completeLatch.countDown(); + } + finally { + locker.unlock(); + } + } + + void awaitBarrier( CyclicBarrier startBarrier )( Stage stage ) { + if ( stage == Stage.beforeMeasureRound ) { + startBarrier.await(); + } + } + + throws( `class AssertionError`, "size of [[benches]] has to be equal to size of [[parameter]]") + shared actual StatisticSummary execute ( + "Options the bench has to be executed with." Options options, + "Number of threads the [[benches]] has to be executed with." [Integer+] parameter + ) { + "Number of executed functions has to be equal to size of the number of threads list." + assert( benches.size == parameter.size ); + + // initialize summary statistic + min = infinity; + max = -infinity; + mean = 0.0; + variance = 0.0; + size = 0; + + Integer totalThreads = sum( parameter ); + // threads completion count down latch + CountDownLatch completeLatch = CountDownLatch( totalThreads ); + // aggregates results + Anything(StatisticSummary) completion = writeResults( completeLatch ); + // append to callback barrier awaiter before measure round - synchronizes measure round start + Options passedOptions = appendCallbacksToOptions( options, awaitBarrier( CyclicBarrier( totalThreads ) ) ); + + // run benches is separated threads + for ( i in 0 : benches.size ) { + assert ( exists threads = parameter[i] ); + assert ( exists bench = benches[i] ); + for ( j in 0 : threads ) { + Thread ( + SyncBench ( + title + i.string + ":" + j.string, bench, completion, passedOptions, [] + ) + ).start(); + } + } + // await bench completion + completeLatch.await(); + + // return results + return StatisticSummary( min, max, mean, Math.sqrt( variance ), size ); + } + +} diff --git a/source/herd/asynctest/benchmark/SyncBench.ceylon b/source/herd/asynctest/benchmark/SyncBench.ceylon new file mode 100644 index 0000000..5d4f78a --- /dev/null +++ b/source/herd/asynctest/benchmark/SyncBench.ceylon @@ -0,0 +1,27 @@ +import java.lang { + Runnable +} + + +"Bench with start synchronization and completion notification." +since( "0.7.0" ) by( "Lis" ) +class SyncBench ( + "Bench title. Generally unique." + String title, + "Function to be tested." + shared actual Anything(*Parameter) bench, + "Completed notification." + void completed( StatisticSummary stat ), + "Options the bench has to be executed with." + Options options, + "Parameters the bench has to be executed with." + Parameter parameter +) + extends BaseBench( title ) + satisfies Runnable + given Parameter satisfies Anything[] +{ + + shared actual void run() => completed( execute( options, parameter ) ); + +} From 0d9991f51941a5ef24797b62b278536b30d3f814 Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 15 Jan 2017 18:31:17 +0300 Subject: [PATCH 59/72] doc --- source/herd/asynctest/benchmark/benchmark.ceylon | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/herd/asynctest/benchmark/benchmark.ceylon b/source/herd/asynctest/benchmark/benchmark.ceylon index ef8f18e..fc78f08 100644 --- a/source/herd/asynctest/benchmark/benchmark.ceylon +++ b/source/herd/asynctest/benchmark/benchmark.ceylon @@ -7,12 +7,16 @@ import java.lang { } +"Clears black hole and resets criteria." +since( "0.7.0" ) by( "Lis" ) void initializeBench( Options options ) { blackHole.clear(); options.measureCriterion.reset(); options.warmupCriterion?.reset(); } +"Resets criteria and verifies black hole." +since( "0.7.0" ) by( "Lis" ) void completeBench( Options options ) { options.measureCriterion.reset(); options.warmupCriterion?.reset(); @@ -20,6 +24,8 @@ void completeBench( Options options ) { blackHole.verifyNumbers(); } +"Runs garbage collector and sleeps for a short amount of time." +since( "0.7.0" ) by( "Lis" ) void warmupBenchmark() { System.gc(); Thread.sleep( 200 ); From 4357dd5265617c2ad78639cdbe970f16d653503d Mon Sep 17 00:00:00 2001 From: Lis Date: Sun, 15 Jan 2017 19:11:08 +0300 Subject: [PATCH 60/72] benchmark example with proxy and metamodel --- .../asynctest/benchmark/proxyAndMeta.ceylon | 84 +++++++++++++++++++ .../herd/examples/asynctest/module.ceylon | 1 + 2 files changed, 85 insertions(+) create mode 100644 examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon diff --git a/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon b/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon new file mode 100644 index 0000000..46e7b92 --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon @@ -0,0 +1,84 @@ +import ceylon.interop.java { + javaClass, + createJavaObjectArray +} +import java.lang.reflect { + Proxy, + Method +} +import java.lang { + ObjectArray +} +import herd.asynctest.benchmark { + benchmark, + writeRelativeToFastest, + Options, + TotalIterations, + SingleBench +} +import herd.asynctest { + AsyncTestContext, + async +} +import ceylon.test { + test +} + + +interface I { + shared formal Integer plusOne( Integer i ); +} + +class II() satisfies I { + shared actual Integer plusOne( Integer i ) => i + 1; +} + + +I instanceDecl() { + return object satisfies I { + II ii = II(); + value f = `I.plusOne`.bind( ii ); + shared actual Integer plusOne( Integer i ) { + return f.apply( i ); + } + }; +} + + +T proxyInstance( T delegate ) given T satisfies Object { + assert ( + is T ret = Proxy.newProxyInstance ( + javaClass().classLoader, + createJavaObjectArray( {javaClass()} ), + ( Object? proxyObject, Method method, ObjectArray? objectArray ) { + method.accessible = true; + if ( exists objectArray ) { + return method.invoke( delegate, *objectArray.iterable ); + //return method.invoke( delegate, *objectArray ); + } + else { + return method.invoke( delegate ); + } + } + ) + ); + return ret; +} + + +"Runs comparative benchmark of direct function call, call via metamodel and call via java proxy." +shared test async void proxyAndMetaBenchmark(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options(TotalIterations(10000), TotalIterations(2000)), + [ + SingleBench("direct", II().plusOne), + SingleBench("meta", instanceDecl().plusOne), + SingleBench("proxy", proxyInstance(II()).plusOne) + ], + [1], [2], [3] + ) + ); + context.complete(); +} diff --git a/examples/herd/examples/asynctest/module.ceylon b/examples/herd/examples/asynctest/module.ceylon index d527cfd..c3beb89 100644 --- a/examples/herd/examples/asynctest/module.ceylon +++ b/examples/herd/examples/asynctest/module.ceylon @@ -44,4 +44,5 @@ module herd.examples.asynctest "0.7.0" { shared import ceylon.test "1.3.1"; shared import herd.asynctest "0.7.0"; shared import java.base "8"; + import ceylon.interop.java "1.3.1"; } From 93803e5eac934616d83e80434b21da3b99ec2f34 Mon Sep 17 00:00:00 2001 From: Lis Date: Thu, 2 Feb 2017 20:03:27 +0300 Subject: [PATCH 61/72] rearranged benchmark --- .../asynctest/benchmark/LockFreeQueue.ceylon | 45 ++++++ .../benchmark/LockFreeQueueBenchmark.ceylon | 43 ++++++ .../benchmark/isValueOrFunction.ceylon | 4 +- .../benchmark/isWrapperOrFunction.ceylon | 32 ++--- .../asynctest/benchmark/plusMinus.ceylon | 6 +- .../asynctest/benchmark/proxyAndMeta.ceylon | 4 +- .../herd/asynctest/benchmark/BaseBench.ceylon | 130 ----------------- .../herd/asynctest/benchmark/BenchFlow.ceylon | 41 ++++++ source/herd/asynctest/benchmark/Clock.ceylon | 120 ++++++++++------ .../benchmark/CompletionCriterion.ceylon | 18 +-- .../herd/asynctest/benchmark/Criteria.ceylon | 117 +++++++++++++++ .../benchmark/EmptyParameterBench.ceylon | 25 ---- .../asynctest/benchmark/ErrorCriteria.ceylon | 92 ------------ .../asynctest/benchmark/GCStrategy.ceylon | 38 ----- .../benchmark/IterationsCriteria.ceylon | 59 -------- .../asynctest/benchmark/MultiBench.ceylon | 101 ++----------- .../herd/asynctest/benchmark/Options.ceylon | 35 ++--- .../asynctest/benchmark/SingleBench.ceylon | 96 ++++++++++++- source/herd/asynctest/benchmark/Stage.ceylon | 53 ------- .../herd/asynctest/benchmark/Statistic.ceylon | 48 +++++++ .../benchmark/StatisticAggregator.ceylon | 32 ++--- .../benchmark/StatisticSummary.ceylon | 40 +----- .../herd/asynctest/benchmark/SyncBench.ceylon | 27 ---- .../benchmark/ThreadLocalClock.ceylon | 85 ----------- .../benchmark/ThreadableRunner.ceylon | 135 ++++++++++++++++++ .../asynctest/benchmark/TimeCriteria.ceylon | 125 ---------------- .../herd/asynctest/benchmark/benchmark.ceylon | 31 ++-- .../asynctest/benchmark/contextWriters.ceylon | 12 +- .../asynctest/benchmark/numberOfGCRuns.ceylon | 11 ++ .../internal/StatisticCalculator.ceylon | 6 + 30 files changed, 704 insertions(+), 907 deletions(-) create mode 100644 examples/herd/examples/asynctest/benchmark/LockFreeQueue.ceylon create mode 100644 examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon delete mode 100644 source/herd/asynctest/benchmark/BaseBench.ceylon create mode 100644 source/herd/asynctest/benchmark/BenchFlow.ceylon create mode 100644 source/herd/asynctest/benchmark/Criteria.ceylon delete mode 100644 source/herd/asynctest/benchmark/EmptyParameterBench.ceylon delete mode 100644 source/herd/asynctest/benchmark/ErrorCriteria.ceylon delete mode 100644 source/herd/asynctest/benchmark/GCStrategy.ceylon delete mode 100644 source/herd/asynctest/benchmark/IterationsCriteria.ceylon delete mode 100644 source/herd/asynctest/benchmark/Stage.ceylon create mode 100644 source/herd/asynctest/benchmark/Statistic.ceylon delete mode 100644 source/herd/asynctest/benchmark/SyncBench.ceylon delete mode 100644 source/herd/asynctest/benchmark/ThreadLocalClock.ceylon create mode 100644 source/herd/asynctest/benchmark/ThreadableRunner.ceylon delete mode 100644 source/herd/asynctest/benchmark/TimeCriteria.ceylon create mode 100644 source/herd/asynctest/benchmark/numberOfGCRuns.ceylon diff --git a/examples/herd/examples/asynctest/benchmark/LockFreeQueue.ceylon b/examples/herd/examples/asynctest/benchmark/LockFreeQueue.ceylon new file mode 100644 index 0000000..b2b959b --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/LockFreeQueue.ceylon @@ -0,0 +1,45 @@ +import java.util.concurrent.atomic { + AtomicReference +} + + +"Lock free queue." +shared class LockFreeQueue() given Item satisfies Object { + + class Node ( + shared variable Item? item = null, + Node? nextNode = null + ) { + shared AtomicReference next = AtomicReference( nextNode ); + } + + + AtomicReference head = AtomicReference( Node() ); + AtomicReference tail = AtomicReference( head.get() ); + + shared void enqueue( Item item ) { + Node newTail = Node( item ); + variable Node oldTail = tail.get(); + while ( !oldTail.next.compareAndSet( null, newTail ) ) { + oldTail = tail.get(); + } + // adjust tail + tail.compareAndSet( oldTail, newTail ); + } + + shared Item? dequeue() { + // current head + variable Node oldHead = head.get(); + // store next in order to retriev correct item even if next will be replaced + variable Node? next = oldHead.next.get(); + // shift head to the next + while ( next exists, !head.compareAndSet( oldHead, next ) ) { + oldHead = head.get(); + next = oldHead.next.get(); + } + return next?.item; + } + + shared Boolean empty => !head.get().next.get() exists; + +} diff --git a/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon b/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon new file mode 100644 index 0000000..cf043ac --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon @@ -0,0 +1,43 @@ +import ceylon.test { + + test +} +import herd.asynctest { + + async, + AsyncTestContext +} +import herd.asynctest.benchmark { + + writeRelativeToFastest, + benchmark, + NumberOfLoops, + Options, + MultiBench, + TimeUnit, + CPUClock +} + + +"[[LockFreeQueue]] multithread producer-consumer benchmark." +shared test async void lockFreeQueueProducerConsumer(AsyncTestContext context) { + // queue to test + LockFreeQueue queue = LockFreeQueue(); + for ( i in 0 : 100 ) { queue.enqueue( 1 ); } + + writeRelativeToFastest ( + context, + benchmark ( + Options(NumberOfLoops(500), NumberOfLoops(100), 100, TimeUnit.milliseconds, CPUClock, true), + [ + MultiBench( + "producer-consumer", + () => queue.enqueue( 1 ), + queue.dequeue + ) + ], + [1, 1], [2, 1], [1, 2], [2, 2] + ) + ); + context.complete(); +} diff --git a/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon index e4a081c..d7b5d47 100644 --- a/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon +++ b/examples/herd/examples/asynctest/benchmark/isValueOrFunction.ceylon @@ -10,7 +10,7 @@ import herd.asynctest.benchmark { benchmark, SingleBench, Options, - TotalIterations + NumberOfLoops } @@ -51,7 +51,7 @@ shared test async void isValueOrFunction(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(TotalIterations(10000), TotalIterations(2000)), + Options(NumberOfLoops(1000), NumberOfLoops(100)), [ SingleBench("value", isValue), SingleBench("function", isFunction) diff --git a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon index 5ffbd5d..d0990a5 100644 --- a/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon +++ b/examples/herd/examples/asynctest/benchmark/isWrapperOrFunction.ceylon @@ -8,10 +8,10 @@ import ceylon.test { import herd.asynctest.benchmark { benchmark, writeRelativeToFastest, - EmptyParameterBench, + SingleBench, TimeUnit, Options, - TotalIterations, + NumberOfLoops, CPUClock, WallClock } @@ -68,29 +68,29 @@ shared test async void isWrapperOrFunctionCPU(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, CPUClock()), + Options(NumberOfLoops(1000), NumberOfLoops(100), 10, TimeUnit.milliseconds, CPUClock), [ - EmptyParameterBench ( + SingleBench<> ( "narrowed single argument wrapper", narrowSingleWrapperToInteger(SingleArgumentWrapper(integerArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "not narrowed single argument wrapper", narrowSingleWrapperToInteger(SingleArgumentWrapper(floatArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "narrowed multiple argument wrapper", narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Integer]>(integerArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "not narrowed multiple argument wrapper", narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Float]>(floatArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "narrowed function", narrowFunctionToInteger(integerArgumentFunction) ), - EmptyParameterBench ( + SingleBench<> ( "not narrowed function", narrowFunctionToInteger(floatArgumentFunction) ) @@ -112,29 +112,29 @@ shared test async void isWrapperOrFunctionWall(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(TotalIterations(10000), TotalIterations(2000), TimeUnit.milliseconds, WallClock()), + Options(NumberOfLoops(1000), NumberOfLoops(100), 10, TimeUnit.milliseconds, WallClock), [ - EmptyParameterBench ( + SingleBench<> ( "narrowed single argument wrapper", narrowSingleWrapperToInteger(SingleArgumentWrapper(integerArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "not narrowed single argument wrapper", narrowSingleWrapperToInteger(SingleArgumentWrapper(floatArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "narrowed multiple argument wrapper", narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Integer]>(integerArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "not narrowed multiple argument wrapper", narrowMultipleWrapperToInteger(MultipleArgumentWrapper<[Float]>(floatArgumentFunction)) ), - EmptyParameterBench ( + SingleBench<> ( "narrowed function", narrowFunctionToInteger(integerArgumentFunction) ), - EmptyParameterBench ( + SingleBench<> ( "not narrowed function", narrowFunctionToInteger(floatArgumentFunction) ) diff --git a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon index ae41651..5aa3df4 100644 --- a/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon +++ b/examples/herd/examples/asynctest/benchmark/plusMinus.ceylon @@ -11,10 +11,10 @@ import herd.asynctest.benchmark { writeRelativeToFastest, benchmark, - LocalIterations, + NumberOfLoops, Options, SingleBench, - LocalError + ErrorCriterion } @@ -29,7 +29,7 @@ shared test async void plusMinusBenchmark(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(LocalIterations(50000).or(LocalError(0.001)), LocalIterations(5000).or(LocalError(0.001))), + Options(NumberOfLoops(1000).or(ErrorCriterion(0.002)), NumberOfLoops(100).or(ErrorCriterion(0.002))), [SingleBench("plus", plusBenchmarkFunction), SingleBench("minus", minusBenchmarkFunction)], [1, 1], [2, 3], [25, 34] diff --git a/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon b/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon index 46e7b92..54c134b 100644 --- a/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon +++ b/examples/herd/examples/asynctest/benchmark/proxyAndMeta.ceylon @@ -13,7 +13,7 @@ import herd.asynctest.benchmark { benchmark, writeRelativeToFastest, Options, - TotalIterations, + NumberOfLoops, SingleBench } import herd.asynctest { @@ -71,7 +71,7 @@ shared test async void proxyAndMetaBenchmark(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(TotalIterations(10000), TotalIterations(2000)), + Options(NumberOfLoops(1000), NumberOfLoops(100)), [ SingleBench("direct", II().plusOne), SingleBench("meta", instanceDecl().plusOne), diff --git a/source/herd/asynctest/benchmark/BaseBench.ceylon b/source/herd/asynctest/benchmark/BaseBench.ceylon deleted file mode 100644 index 865f1b3..0000000 --- a/source/herd/asynctest/benchmark/BaseBench.ceylon +++ /dev/null @@ -1,130 +0,0 @@ -import java.lang.management { - ManagementFactory -} - - -"Intended to implement details of bench execution in a signle thread. - - Just [[bench]] has to be implemented which is indended to invoke test function once. - See [[execute]] for the run cycle. - " -tagged( "Bench" ) -since( "0.7.0" ) by( "Lis" ) -shared abstract class BaseBench ( - shared actual String title -) - satisfies Bench - given Parameter satisfies Anything[] -{ - - variable Integer? memoizedHash = null; - - shared actual default Integer hash => memoizedHash else ( memoizedHash = 7 * title.hash + 37 ); - - - "Executes optioned callbacks." - void stageEvent ( - "Stage the event belongs to." Stage stage, "Callbacks." Anything(Stage)[] callbacks - ) { - for ( item in callbacks ) { - item( stage ); - } - } - - - "Bench test function. - May return results in order to avoid dead-code elimination. - Returned result is consumed using [[pushToBlackHole]]." - shared formal Anything(*Parameter) bench; - - - "Returns number of GC runs up to now." - Integer numberOfGCRuns() { - value gcBeans = ManagementFactory.garbageCollectorMXBeans; - variable Integer numGC = 0; - for ( gcBean in gcBeans ) { numGC += gcBean.collectionCount; } - return numGC; - } - - "Executes test using the following cycle: - * [[stageEvent]] with [[Stage.beforeWarmupRound]] - * Cycle while [[Options.warmupCriterion]] is not met: - * [[stageEvent]] with [[Stage.beforeWarmupIteration]] - * [[bench]] - * consume result returned by [[bench]] using [[pushToBlackHole]] - * [[stageEvent]] with [[Stage.afterWarmupIteration]] - * [[stageEvent]] with [[Stage.afterWarmupRound]] - * [[stageEvent]] with [[Stage.beforeMeasureRound]] - * Cycle while [[Options.measureCriterion]] is not met: - * [[stageEvent]] with [[Stage.beforeMeasureIteration]] - * [[bench]] - * collect time statistic - * consume result returned by [[bench]] using [[pushToBlackHole]] - * [[stageEvent]] with [[Stage.afterMeasureIteration]] - * [[stageEvent]] with [[Stage.afterMeasureRound]] - - > Note: runs when GC wroks are excluded from statistic calculations. - " - shared actual StatisticSummary execute ( - Options options, Parameter parameter - ) { - - // clock to measure time interval - Clock clock = options.clock; - // factor to scale time delta from nanoseconds (measured in) to timeUnit - Float timeFactor = clock.units.factorToSeconds / options.timeUnit.factorToSeconds; - - variable CompletionCriterion? warmupCriterion = options.warmupCriterion; - StatisticAggregator calculator = StatisticAggregator(); - if ( warmupCriterion exists ) { stageEvent( Stage.beforeWarmupRound, options.callbacks ); } - else { stageEvent( Stage.beforeMeasureRound, options.callbacks ); } - - // bench iterations - while ( true ) { - // number of GC starts before test run - Integer numGCBefore = numberOfGCRuns(); - // before iteration event - if ( warmupCriterion exists ) { stageEvent( Stage.beforeWarmupIteration, options.callbacks ); } - else { stageEvent( Stage.beforeMeasureIteration, options.callbacks ); } - // execute the test function - clock.start(); - Anything ret = bench( *parameter ); - Float delta = clock.measure() * timeFactor; - // calculate execution statistic - if ( delta > 0.0 ) { - // number of GC starts after test run - Integer numGCAfter = numberOfGCRuns(); - if ( numGCAfter == numGCBefore || !options.skipGCRuns ) { - // add sample only if GC has not been started during the test or GC runs have not be skipped - calculator.sample( 1.0 / delta ); - if ( exists criterion = warmupCriterion ) { - if ( criterion.verify( delta, calculator, options.timeUnit ) ) { - // warmup round is completed - warmupCriterion = null; - calculator.reset(); - stageEvent( Stage.afterWarmupIteration, options.callbacks ); - stageEvent( Stage.afterWarmupRound, options.callbacks ); - stageEvent( Stage.beforeMeasureRound, options.callbacks ); - continue; - } - } - else { - if ( options.measureCriterion.verify( delta, calculator, options.timeUnit ) ) { - // measure round is completed - stageEvent( Stage.afterMeasureIteration, options.callbacks ); - stageEvent( Stage.afterMeasureRound, options.callbacks ); - break; - } - } - } - } - // completing iteration - if ( warmupCriterion exists ) { stageEvent( Stage.afterWarmupIteration, options.callbacks ); } - else { stageEvent( Stage.afterMeasureIteration, options.callbacks ); } - pushToBlackHole( ret ); - } - - return calculator.result; - } - -} diff --git a/source/herd/asynctest/benchmark/BenchFlow.ceylon b/source/herd/asynctest/benchmark/BenchFlow.ceylon new file mode 100644 index 0000000..e270dfd --- /dev/null +++ b/source/herd/asynctest/benchmark/BenchFlow.ceylon @@ -0,0 +1,41 @@ + + +"Represents benchmark function + test flow. I.e. functions called before / after test iteration." +tagged( "Bench" ) +see( `function benchmark`, `class SingleBench`, `class MultiBench` ) +since( "0.7.0" ) by( "Lis" ) +shared interface BenchFlow + given Parameter satisfies Anything[] +{ + + "Setups the benchmark test. Called before the test." + shared formal void setup(); + + "Called before each [[bench]] execution." + shared formal void before(); + + "Benchmark function." + shared formal Anything(*Parameter) bench; + + "Called after each [[bench]] execution." + shared formal void after(); + + "Disposes the benchmark test. Called after the test." + shared formal void dispose(); + +} + + +"Just [[bench]] function makes sense." +since( "0.7.0" ) by( "Lis" ) +class EmptyBenchFlow ( + shared actual Anything(*Parameter) bench +) + satisfies BenchFlow + given Parameter satisfies Anything[] +{ + shared actual void after() {} + shared actual void before() {} + shared actual void dispose() {} + shared actual void setup() {} +} diff --git a/source/herd/asynctest/benchmark/Clock.ceylon b/source/herd/asynctest/benchmark/Clock.ceylon index 2553e77..643058c 100644 --- a/source/herd/asynctest/benchmark/Clock.ceylon +++ b/source/herd/asynctest/benchmark/Clock.ceylon @@ -1,8 +1,5 @@ -import java.lang { - ThreadLocal -} -import java.util.\ifunction { - Supplier +import java.lang.management { + ManagementFactory } @@ -18,7 +15,7 @@ import java.util.\ifunction { for ( item in 0 : totalRuns ) { clock.start(); doOperation(); - Integer interval = clock.measure(); + Integer interval = clock.measure( TimeUnit.seconds ); } Time interval is measured relative to the thread current to the caller. @@ -31,14 +28,12 @@ since( "0.7.0" ) by( "Lis" ) shared interface Clock { - "Units the clock measures time interval in." - shared formal TimeUnit units; - "Starts time interval measure from the current time and locally for the current thread." shared formal void start(); - "Measures time interval from the last [[start]] call (in the same thread as `measure` called) and up to now." - shared formal Integer measure(); + "Measures time interval from the last [[start]] call (in the same thread as `measure` called) and up to now. + Returns time interval in the given time units [[units]]." + shared formal Float measure( "Units the time interval is to be measured in." TimeUnit units ); } @@ -49,61 +44,92 @@ see( `class Options`, `class CPUClock`, `class UserClock` ) since( "0.7.0" ) by( "Lis" ) shared class WallClock() satisfies Clock { - ThreadLocal startWallTime = ThreadLocal(); + variable Integer startWallTime = 0; + + shared actual void start() { + startWallTime = system.nanoseconds; + } + + shared actual Float measure( TimeUnit units ) + => TimeUnit.nanoseconds.factorToSeconds / units.factorToSeconds * ( system.nanoseconds - startWallTime ); + + shared actual String string => "wall clock"; +} + + +"Base for the CPU time interval measurements. + Calculates a factor to scale wall time interval corresponds to measure request. + The factor is moving averaged ratio of CPU time interval to wall time interval. + Ends of CPU time interval are measured as close as possible to ends of wall time interval. + + [[readCurrentTime]] provides particular implementation of CPU or user times. + " +since( "0.7.0" ) by( "Lis" ) +shared abstract class ThreadClock() of CPUClock | UserClock satisfies Clock { + + variable Integer startWallTime = system.nanoseconds; + variable Integer lastWallTime = 0; + variable Integer lastSample = 0; + variable Float factor = 1.0; + + if ( !ManagementFactory.threadMXBean.threadCpuTimeEnabled ) { + ManagementFactory.threadMXBean.threadCpuTimeEnabled = true; + } - shared actual TimeUnit units => TimeUnit.nanoseconds; + "Returns current time based on strategy." + shared formal Integer readCurrentTime(); shared actual void start() { - startWallTime.set( system.nanoseconds ); + Integer currentSample = readCurrentTime(); + Integer currentWallTime = system.nanoseconds; + if ( lastSample > 0 && currentSample > lastSample && currentWallTime > lastWallTime ) { + // Moving averaging factor! + factor = 0.9 * factor + 0.1 * ( currentSample - lastSample ) / ( currentWallTime - lastWallTime ); + lastSample = currentSample; + lastWallTime = currentWallTime; + } + startWallTime = system.nanoseconds; } - shared actual Integer measure() => system.nanoseconds - startWallTime.get(); + shared actual Float measure( TimeUnit units ) + => TimeUnit.nanoseconds.factorToSeconds / units.factorToSeconds * factor * ( system.nanoseconds - startWallTime ); - shared actual String string => "wall clock"; } -"CPU clock uses `ThreadMXBean.currentThreadCpuTime` to measure time interval." +"CPU clock uses `ThreadMXBean.currentThreadCpuTime` to measure time interval. + If thread CPU time is not supported measures wall time interval." tagged( "Clock" ) see( `class Options`, `class UserClock`, `class WallClock` ) since( "0.7.0" ) by( "Lis" ) -shared class CPUClock() satisfies Clock -{ - - ThreadLocal localClock = ThreadLocal.withInitial ( - object satisfies Supplier { - get() => CPULocalClock(); +shared class CPUClock() extends ThreadClock() { + shared actual Integer readCurrentTime() { + value threadMXBean = ManagementFactory.threadMXBean; + if ( threadMXBean.currentThreadCpuTimeSupported ) { + return threadMXBean.currentThreadCpuTime; } - ); - - shared actual TimeUnit units => TimeUnit.nanoseconds; - - shared actual void start() => localClock.get().start(); - - shared actual Integer measure() => localClock.get().measure(); - + else { + return system.nanoseconds; + } + } shared actual String string => "CPU clock"; } -"User clock uses `ThreadMXBean.currentThreadUserTime` to measure time interval." +"User clock uses `ThreadMXBean.currentThreadUserTime` to measure time interval. + If thread CPU time is not supported measures wall time interval." tagged( "Clock" ) see( `class Options`, `class CPUClock`, `class WallClock` ) since( "0.7.0" ) by( "Lis" ) -shared class UserClock() satisfies Clock -{ - - ThreadLocal localClock = ThreadLocal.withInitial ( - object satisfies Supplier { - get() => UserLocalClock(); +shared class UserClock() extends ThreadClock() { + shared actual Integer readCurrentTime() { + value threadMXBean = ManagementFactory.threadMXBean; + if ( threadMXBean.currentThreadCpuTimeSupported ) { + return threadMXBean.currentThreadUserTime; } - ); - - shared actual TimeUnit units => TimeUnit.nanoseconds; - - shared actual void start() => localClock.get().start(); - - shared actual Integer measure() => localClock.get().measure(); - - shared actual String string => "User clock"; + else { + return system.nanoseconds; + } + } + shared actual String string => "user clock"; } diff --git a/source/herd/asynctest/benchmark/CompletionCriterion.ceylon b/source/herd/asynctest/benchmark/CompletionCriterion.ceylon index f45f80d..222082b 100644 --- a/source/herd/asynctest/benchmark/CompletionCriterion.ceylon +++ b/source/herd/asynctest/benchmark/CompletionCriterion.ceylon @@ -11,13 +11,9 @@ shared interface CompletionCriterion { "Returns `true` if completion criterion is met and `false` otherwise." shared formal Boolean verify ( - "The latest execution time in the given time unit." - Float delta, - "Accumulated execution statistic of operations per given time unit. - Statistic is aggregated by theexecution on the current thread. - This is mutable but be careful to modify aggregated statistic. It may lead to wrong summary results." - StatisticAggregator stat, - "Time unit [[delta]] and [[stat]] are measured in." + "Accumulated execution statistic of operations per the given time unit." + Statistic stat, + "Time unit [[stat]] is measured in." TimeUnit timeUnit ); @@ -46,9 +42,9 @@ class AndCriterion( CompletionCriterion+ criteria ) satisfies CompletionCriterio } } - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { + shared actual Boolean verify( Statistic stat, TimeUnit timeUnit ) { for ( item in criteria ) { - if ( !item.verify( delta, stat, timeUnit ) ) { + if ( !item.verify( stat, timeUnit ) ) { return false; } } @@ -72,9 +68,9 @@ class OrCriterion( CompletionCriterion+ criteria ) satisfies CompletionCriterion } } - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { + shared actual Boolean verify( Statistic stat, TimeUnit timeUnit ) { for ( item in criteria ) { - if ( item.verify( delta, stat, timeUnit ) ) { + if ( item.verify( stat, timeUnit ) ) { return true; } } diff --git a/source/herd/asynctest/benchmark/Criteria.ceylon b/source/herd/asynctest/benchmark/Criteria.ceylon new file mode 100644 index 0000000..bc1739f --- /dev/null +++ b/source/herd/asynctest/benchmark/Criteria.ceylon @@ -0,0 +1,117 @@ + + +"Continues benchmark iterations while total number of benchmark loops doesn't exceed [[numberOfLoops]]. + " +tagged( "Criteria" ) +see( `class Options` ) +throws( `class AssertionError`, "Total number of benchmark loops is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class NumberOfLoops ( + "Total number of benchmark loops. Has to be > 0." Integer numberOfLoops +) + satisfies CompletionCriterion +{ + + "Total number of benchmark loops has to be > 0." + assert( numberOfLoops > 0 ); + + + shared actual void reset() { + } + + shared actual Boolean verify( Statistic stat, TimeUnit timeUnit ) { + return stat.size >= numberOfLoops; + } + +} + + +"Continues benchmark iterations while relative sample error doesn't exceed [[maxRelativeError]]. + I.e. [[StatisticSummary.relativeSampleError]] is compared against [[maxRelativeError]]." +tagged( "Criteria" ) +see( `class Options` ) +throws( `class AssertionError`, "Maximum allowed error is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class ErrorCriterion ( + "Maximum allowed error. Has to be > 0." Float maxRelativeError, + "Minimum number of iterations before check error. Default is 10." Integer minIterations = 10 +) + satisfies CompletionCriterion +{ + + "Maximum allowed error has to be > 0." + assert( maxRelativeError > 0.0 ); + + + shared actual void reset() { + } + + shared actual Boolean verify( Statistic stat, TimeUnit timeUnit ) { + return stat.size > minIterations && stat.relativeSampleError < maxRelativeError; + } + +} + + +"Continues benchmark iterations while total execution time doesn't exceed [[timeLimit]]. + Execution time is overall time spent on test execution. + I.e. it summarizes time spent on benchmark function execution as well as on internal calculations. + + The alternative way is take into account time spent on benchmark function execution only see [[TotalBenchTime]]. + " +tagged( "Criteria" ) +see( `class Options`, `class TotalBenchTime` ) +throws( `class AssertionError`, "Maximum permited execution is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class TotalExecutionTime ( + "Maximum permited execution time in [[timeUnit]] units." Integer timeLimit, + "Unit the [[timeLimit]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds +) + satisfies CompletionCriterion +{ + "Maximum permited execution has to be > 0." + assert( timeLimit > 0 ); + Integer totalNanoseconds = ( timeLimit * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + + variable Integer startTime = 0; + + + shared actual void reset() { + startTime = system.nanoseconds; + } + + shared actual Boolean verify( Statistic stat, TimeUnit deltaTimeUnit ) { + return system.nanoseconds - startTime > totalNanoseconds; + } + +} + + +"Continues benchmark iterations while overall time accumulated by benchmark iterations doesn't exceed + [[totalTime]]. + This criterion summarizes only time spent on benchmark function execution for all execution threads jointly. + " +tagged( "Criteria" ) +see( `class Options`, `class TotalExecutionTime` ) +throws( `class AssertionError`, "Total benchmark time is <= 0." ) +since( "0.7.0" ) by( "Lis" ) +shared class TotalBenchTime ( + "Maximum time (in [[timeUnit]] units) to be accumulated by benchmark iterations." Integer totalTime, + "Unit the [[totalTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds +) + satisfies CompletionCriterion +{ + "Total benchmark time has to be > 0." + assert( totalTime > 0 ); + Integer totalNanoseconds = ( totalTime * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; + + + shared actual void reset() { + } + + shared actual Boolean verify( Statistic stat, TimeUnit deltaTimeUnit ) { + return ( stat.size / stat.mean * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer + > totalNanoseconds; + } + +} diff --git a/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon b/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon deleted file mode 100644 index 3051900..0000000 --- a/source/herd/asynctest/benchmark/EmptyParameterBench.ceylon +++ /dev/null @@ -1,25 +0,0 @@ - -"Executes test function in a single thread. - Test function takes no argument - the same as to `SingleBench<[]>`." -tagged( "Bench" ) -see( `function benchmark`, `class SingleBench` ) -since( "0.7.0" ) by( "Lis" ) -shared final class EmptyParameterBench ( - "Bench title. Generally unique." - String title, - "Function to be tested." - shared actual Anything() bench -) - extends BaseBench<[]>( title ) -{ - - shared actual Boolean equals( Object other ) { - if ( is EmptyParameterBench other ) { - return title == other.title; - } - else { - return false; - } - } - -} diff --git a/source/herd/asynctest/benchmark/ErrorCriteria.ceylon b/source/herd/asynctest/benchmark/ErrorCriteria.ceylon deleted file mode 100644 index 6a62b64..0000000 --- a/source/herd/asynctest/benchmark/ErrorCriteria.ceylon +++ /dev/null @@ -1,92 +0,0 @@ -import java.util.concurrent { - ConcurrentSkipListMap -} -import java.lang { - Thread, - Math -} - - -"Continues benchmark iterations while total (over all threads) relative sample error doesn't exceed [[maxRelativeError]]. - I.e. [[StatisticSummary.relativeSampleError]] is compared against [[maxRelativeError]]." -tagged( "Criteria" ) -see( `class Options`, `class LocalError` ) -throws( `class AssertionError`, "Maximum allowed total error is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class TotalError ( - "Maximum allowed total error. Has to be > 0." Float maxRelativeError, - "Minimum number of iterations before check error. Default is 10." Integer minIterations = 10 -) - satisfies CompletionCriterion -{ - - "Maximum allowed total error has to be > 0." - assert( maxRelativeError > 0.0 ); - - class Stat ( - shared variable Float mean, - shared variable Float variance, - shared variable Integer size - ) {} - - ConcurrentSkipListMap localStat = ConcurrentSkipListMap( IntegerComparator() ); - - - shared actual void reset() { - localStat.clear(); - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { - Integer id = Thread.currentThread().id; - if ( exists localStat = localStat.get( id ) ) { - localStat.mean = stat.mean; - localStat.variance = stat.variance; - localStat.size = stat.size; - } - else { - localStat.put( id, Stat( stat.mean, stat.variance, stat.size ) ); - } - variable Integer size = 0; - variable Float mean = 0.0; - variable Float variance = 0.0; - for ( item in localStat.values() ) { - size += item.size; - mean += item.mean; - variance += item.variance; - } - if ( size > minIterations ) { - return Math.sqrt( variance / ( size - 1 ) ) / mean < maxRelativeError; - } - else { - return false; - } - } - -} - - -"Continues benchmark iterations while thread local relative sample error doesn't exceed [[maxRelativeError]]. - I.e. [[StatisticSummary.relativeSampleError]] is compared against [[maxRelativeError]]." -tagged( "Criteria" ) -see( `class Options`, `class TotalError` ) -throws( `class AssertionError`, "Maximum allowed local error is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class LocalError ( - "Maximum allowed local error. Has to be > 0." Float maxRelativeError, - "Minimum number of iterations before check error. Default is 10." Integer minIterations = 10 -) - satisfies CompletionCriterion -{ - - "Maximum allowed local error has to be > 0." - assert( maxRelativeError > 0.0 ); - - - shared actual void reset() { - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { - return stat.size > minIterations && stat.relativeSampleError < maxRelativeError; - } - -} diff --git a/source/herd/asynctest/benchmark/GCStrategy.ceylon b/source/herd/asynctest/benchmark/GCStrategy.ceylon deleted file mode 100644 index c344a3a..0000000 --- a/source/herd/asynctest/benchmark/GCStrategy.ceylon +++ /dev/null @@ -1,38 +0,0 @@ -import java.lang { - System -} - - -"Executes garbage collector at the given `stage`." -tagged( "Callbacks" ) -see( `class Options` ) -since( "0.7.0" ) by( "Lis" ) -shared void gcCallback( "Stages GC has to be executed at." Stage+ stages )( "Current stage." Stage stage ) { - if ( stage in stages ) { - System.gc(); - } -} - -"Executes garbage collector before warmup and measure rounds. " -tagged( "Callbacks" ) -see( `class Options` ) -since( "0.7.0" ) by( "Lis" ) -shared Anything(Stage) gcBeforeRounds = gcCallback( Stage.beforeWarmupRound, Stage.beforeMeasureRound ); - -"Executes garbage collector before measure round." -tagged( "Callbacks" ) -see( `class Options` ) -since( "0.7.0" ) by( "Lis" ) -shared Anything(Stage) gcBeforeMeasureRound = gcCallback( Stage.beforeMeasureRound ); - -"Executes garbage collector before each measure iteration." -tagged( "Callbacks" ) -see( `class Options` ) -since( "0.7.0" ) by( "Lis" ) -shared Anything(Stage) gcBeforeMeasureIteration = gcCallback( Stage.beforeMeasureIteration ); - -"Executes garbage collector before each warmup and measure iteration." -tagged( "Callbacks" ) -see( `class Options` ) -since( "0.7.0" ) by( "Lis" ) -shared Anything(Stage) gcBeforeIterations = gcCallback( Stage.beforeWarmupIteration, Stage.beforeMeasureIteration ); diff --git a/source/herd/asynctest/benchmark/IterationsCriteria.ceylon b/source/herd/asynctest/benchmark/IterationsCriteria.ceylon deleted file mode 100644 index d3e9c44..0000000 --- a/source/herd/asynctest/benchmark/IterationsCriteria.ceylon +++ /dev/null @@ -1,59 +0,0 @@ -import java.util.concurrent.atomic { - AtomicLong -} - - -"Continues benchmark iterations while total number of iterations doesn't exceed [[numberOfIterations]]. - Number of iterations is equal to number of calls of [[verify]] after last call of [[reset]]. - So, the benchmark is completed when sum of iterations over all used threads reaches [[numberOfIterations]]. - " -tagged( "Criteria" ) -see( `class Options`, `class LocalIterations` ) -throws( `class AssertionError`, "Total number of benchmark iterations is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class TotalIterations ( - "Total number of benchmark iterations. Has to be > 0." Integer numberOfIterations -) - satisfies CompletionCriterion -{ - - "Total number of benchmark iterations has to be > 0." - assert( numberOfIterations > 0 ); - - AtomicLong counter = AtomicLong( 1 ); - - shared actual void reset() { - counter.set( 1 ); - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { - return counter.incrementAndGet() > numberOfIterations; - } - -} - - -"Continues benchmark iterations while local (relative to thread) number of iterations doesn't exceed [[numberOfIterations]]. - Number of iterations is equal to number of calls of [[verify]] after last call of [[reset]]." -tagged( "Criteria" ) -see( `class Options`, `class TotalIterations` ) -throws( `class AssertionError`, "Thread local restriction on a number of benchmark iterations is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class LocalIterations ( - "Thread local restriction on a number of benchmark iterations. Has to be > 0." Integer numberOfIterations -) - satisfies CompletionCriterion -{ - - "Thread local restriction on a number of benchmark iterations has to be > 0." - assert( numberOfIterations > 0 ); - - - shared actual void reset() { - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit timeUnit ) { - return stat.size >= numberOfIterations; - } - -} diff --git a/source/herd/asynctest/benchmark/MultiBench.ceylon b/source/herd/asynctest/benchmark/MultiBench.ceylon index a426acd..6a00400 100644 --- a/source/herd/asynctest/benchmark/MultiBench.ceylon +++ b/source/herd/asynctest/benchmark/MultiBench.ceylon @@ -1,14 +1,3 @@ -import java.util.concurrent.locks { - ReentrantLock -} -import java.util.concurrent { - CyclicBarrier, - CountDownLatch -} -import java.lang { - Thread, - Math -} "Executes bench functions `benches` in separated threads. @@ -22,84 +11,22 @@ tagged( "Bench" ) since( "0.7.0" ) by( "Lis" ) shared class MultiBench ( shared actual String title, - "Bench functions." Anything()+ benches + "Bench functions." Anything() | BenchFlow<[]> + benchFunctions ) satisfies Bench<[Integer+]> -{ - - ReentrantLock locker = ReentrantLock(); - variable Float min = infinity; - variable Float max = -infinity; - variable Float mean = 0.0; - variable Float variance = 0.0; - variable Integer size = 0; - - - void writeResults( CountDownLatch completeLatch )( StatisticSummary stat ) { - locker.lock(); - try { - if ( stat.min < min ) { - min = stat.min; - } - if ( stat.max > max ) { - max = stat.max; - } - mean += stat.mean; - variance += stat.variance; - size += stat.size; - completeLatch.countDown(); - } - finally { - locker.unlock(); - } - } - - void awaitBarrier( CyclicBarrier startBarrier )( Stage stage ) { - if ( stage == Stage.beforeMeasureRound ) { - startBarrier.await(); - } - } +{ - throws( `class AssertionError`, "size of [[benches]] has to be equal to size of [[parameter]]") - shared actual StatisticSummary execute ( - "Options the bench has to be executed with." Options options, - "Number of threads the [[benches]] has to be executed with." [Integer+] parameter - ) { - "Number of executed functions has to be equal to size of the number of threads list." - assert( benches.size == parameter.size ); - - // initialize summary statistic - min = infinity; - max = -infinity; - mean = 0.0; - variance = 0.0; - size = 0; - - Integer totalThreads = sum( parameter ); - // threads completion count down latch - CountDownLatch completeLatch = CountDownLatch( totalThreads ); - // aggregates results - Anything(StatisticSummary) completion = writeResults( completeLatch ); - // append to callback barrier awaiter before measure round - synchronizes measure round start - Options passedOptions = appendCallbacksToOptions( options, awaitBarrier( CyclicBarrier( totalThreads ) ) ); - - // run benches is separated threads - for ( i in 0 : benches.size ) { - assert ( exists threads = parameter[i] ); - assert ( exists bench = benches[i] ); - for ( j in 0 : threads ) { - Thread ( - SyncBench ( - title + i.string + ":" + j.string, bench, completion, passedOptions, [] - ) - ).start(); - } - } - // await bench completion - completeLatch.await(); - - // return results - return StatisticSummary( min, max, mean, Math.sqrt( variance ), size ); - } + shared actual StatisticSummary execute( Options options, [Integer+] parameter ) + => object extends ThreadableRunner( options ) { + variable RunnableBench[] benchesMemo = []; + shared actual RunnableBench[] benches + => if ( benchesMemo nonempty ) then benchesMemo + else ( benchesMemo = [ + for ( [func, threadNum] in zipPairs( benchFunctions, parameter ) ) + for ( i in 0 : threadNum ) + RunnableBench( if ( is BenchFlow<[]> func ) then func else EmptyBenchFlow( func ) ) + ] + ); + }.execute(); } diff --git a/source/herd/asynctest/benchmark/Options.ceylon b/source/herd/asynctest/benchmark/Options.ceylon index 2fd5196..5de051a 100644 --- a/source/herd/asynctest/benchmark/Options.ceylon +++ b/source/herd/asynctest/benchmark/Options.ceylon @@ -10,28 +10,15 @@ shared class Options ( "Optional criterion which identifies when warmup round have to be completed. Warmup round is skipped if `null`." shared CompletionCriterion? warmupCriterion = null, - "Time unit the results have to be reported with. Default is seconds." + "Number of iterations per a one loop. + Total number of loops are identified by [[warmupCriterion]] for warmup round + and by [[measureCriterion]] for measure round." + shared Integer iterationsPerLoop = 10, + "Time unit the results have to be reported with." shared TimeUnit timeUnit = TimeUnit.seconds, - "Clock to measure time intervals. Default wall clock." - shared Clock clock = WallClock(), - "`true` if runs when GC has been started has to be skipped and `flase` otherwise. - By default is `true`." - shared Boolean skipGCRuns = true, - "Callbacks executed at each stage. - By default callback which executes garbage collector before warmup and measure rounds (see [[gcBeforeRounds]])." - shared Anything(Stage)[] callbacks = [gcBeforeRounds] -) { -} - - -"Returns options with prepended callbacks." -since( "0.7.0" ) by( "Lis" ) -Options prependCallbacksToOptions( Options options, Anything(Stage)+ callbacks ) - => Options( options.measureCriterion, options.warmupCriterion, options.timeUnit, options.clock, options.skipGCRuns, - options.callbacks.prepend( callbacks ) ); - -"Returns options with appended callbacks." -since( "0.7.0" ) by( "Lis" ) -Options appendCallbacksToOptions( Options options, Anything(Stage)+ callbacks ) - => Options( options.measureCriterion, options.warmupCriterion, options.timeUnit, options.clock, options.skipGCRuns, - options.callbacks.append( callbacks ) ); + "Clock factory (instantiated for each bench and each execution thread ). + The clock is used to measure time intervals." + shared Clock() clock = WallClock, + "`true` if runs when GC has been started has to be skipped and `flase` otherwise." + shared Boolean skipGCRuns = true +) {} diff --git a/source/herd/asynctest/benchmark/SingleBench.ceylon b/source/herd/asynctest/benchmark/SingleBench.ceylon index 8695742..5d84779 100644 --- a/source/herd/asynctest/benchmark/SingleBench.ceylon +++ b/source/herd/asynctest/benchmark/SingleBench.ceylon @@ -1,18 +1,28 @@ +import java.lang { + System +} + "Executes test function in a single thread." tagged( "Bench" ) -see( `function benchmark`, `class EmptyParameterBench` ) +see( `function benchmark` ) since( "0.7.0" ) by( "Lis" ) -shared final class SingleBench ( +shared final class SingleBench ( "Bench title. Generally unique." - String title, + shared actual String title, "Function to be tested." - shared actual Anything(*Parameter) bench + Anything(*Parameter) | BenchFlow bench ) - extends BaseBench( title ) + satisfies Bench given Parameter satisfies Anything[] { + BenchFlow actualBench = if ( is BenchFlow bench ) then bench else EmptyBenchFlow( bench ); + + variable Integer? memoizedHash = null; + + shared actual Integer hash => memoizedHash else ( memoizedHash = 7 * title.hash + 37 ); + shared actual Boolean equals( Object other ) { if ( is SingleBench other ) { return title == other.title; @@ -22,4 +32,80 @@ shared final class SingleBench ( } } + "Executes test using the following cycle: + * Cycle while [[Options.warmupCriterion]] is not met: + * Cycle of [[Options.iterationsPerLoop]]: + * [[bench]] + * Consume result returned by [[bench]] using [[pushToBlackHole]] + * Cycle while [[Options.measureCriterion]] is not met: + * Cycle of [[Options.iterationsPerLoop]]: + * [[bench]] + * Increment time spent on [[bench]] execution + * Consume result returned by [[bench]] using [[pushToBlackHole]] + * Collect statistic in operations per time units + " + shared actual StatisticSummary execute ( + "Options the bench is executed with." Options options, + "Execution parameter the benchmark function takes." Parameter parameter + ) { + + // clock to measure time interval + Clock clock = options.clock(); + + variable CompletionCriterion? warmupCriterion = options.warmupCriterion; + StatisticAggregator calculator = StatisticAggregator(); + System.gc(); + + actualBench.setup(); + // bench iterations + while ( true ) { + // test loop + variable Float loopTime = 0.0; + variable Integer loop = options.iterationsPerLoop; + while ( loop -- > 0 ) { + actualBench.before(); + // number of GC starts before test run + Integer numGCBefore = numberOfGCRuns(); + // execute the test function + clock.start(); + Anything ret = actualBench.bench( *parameter ); + Float delta = clock.measure( options.timeUnit ); + // number of GC starts after test run + Integer numGCAfter = numberOfGCRuns(); + if ( numGCAfter == numGCBefore || !options.skipGCRuns ) { + loopTime += delta; + } + else { + loop ++; + } + // Eleminate JIT optimization + pushToBlackHole( ret ); + actualBench.after(); + } + + // calculate execution statistic + if ( loopTime > 0.0 ) { + // add sample only if GC has not been started during the test or GC runs have not be skipped + calculator.sample( options.iterationsPerLoop / loopTime ); + if ( exists criterion = warmupCriterion ) { + if ( criterion.verify( calculator, options.timeUnit ) ) { + // warmup round is completed + warmupCriterion = null; + calculator.reset(); + System.gc(); + } + } + else { + if ( options.measureCriterion.verify( calculator, options.timeUnit ) ) { + // measure round is completed + break; + } + } + } + } + + actualBench.dispose(); + return calculator.result; + } + } diff --git a/source/herd/asynctest/benchmark/Stage.ceylon b/source/herd/asynctest/benchmark/Stage.ceylon deleted file mode 100644 index cf19d76..0000000 --- a/source/herd/asynctest/benchmark/Stage.ceylon +++ /dev/null @@ -1,53 +0,0 @@ - - -"Identifies a stage of the bench execution." -see( `class Options`, `function gcCallback` ) -tagged( "Options" ) -since( "0.7.0" ) by( "Lis" ) -shared class Stage - of beforeWarmupRound | afterWarmupRound | beforeWarmupIteration | afterWarmupIteration - | beforeMeasureRound | afterMeasureRound | beforeMeasureIteration | afterMeasureIteration -{ - shared actual String string; - - "Before warmup round." - shared new beforeWarmupRound { - string = "before warmup round stage"; - } - - "After warmup round." - shared new afterWarmupRound { - string = "after warmup round stage"; - } - - "Before warmup iteration." - shared new beforeWarmupIteration { - string = "before warmup iteration stage"; - } - - "After warmup iteration." - shared new afterWarmupIteration { - string = "after warmup iteration stage"; - } - - "Before measure round." - shared new beforeMeasureRound { - string = "before measure round stage"; - } - - "After measure round." - shared new afterMeasureRound { - string = "after measure round stage"; - } - - "Before measure iteration." - shared new beforeMeasureIteration { - string = "before measure iteration stage"; - } - - "After measure iteration." - shared new afterMeasureIteration { - string = "after measure iteration stage"; - } - -} diff --git a/source/herd/asynctest/benchmark/Statistic.ceylon b/source/herd/asynctest/benchmark/Statistic.ceylon new file mode 100644 index 0000000..4a2b240 --- /dev/null +++ b/source/herd/asynctest/benchmark/Statistic.ceylon @@ -0,0 +1,48 @@ +import java.lang { + Math +} + + +"Provides statistic data for a stream of variate values." +tagged( "Result" ) +see( `class StatisticSummary`, `class StatisticAggregator` ) +since( "0.6.0" ) by( "Lis" ) +shared interface Statistic { + + "Minimum of the values that have been statisticaly treated." + shared formal Float min; + + "Maximum of the values that have been statisticaly treated." + shared formal Float max; + + "Mean value." + shared formal Float mean; + + "Returns standard deviation of the values that have been statisticaly treated. + Standard deviation is `variance^0.5`." + shared formal Float standardDeviation; + + "The number of the values that have been statisticaly treated." + shared formal Integer size; + + + "Variance of the values that have been statisticaly treated. + The variance is mean((x-mean(x))^2)." + shared default Float variance => standardDeviation * standardDeviation; + + "Sample variance is size/(size - 1)*variance." + shared default Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; + + "Sample standard deviation is sqrt(size/(size - 1)*variance)." + shared default Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) ) * standardDeviation else 0.0; + + "Equals to standardDeviation/sqrt(size)." + shared default Float standardError => if ( size > 1 ) then standardDeviation / Math.sqrt( size.float ) else 0.0; + + "Equals to sampleStandardDeviation/sqrt(size)." + shared default Float sampleError => if ( size > 1 ) then standardDeviation / Math.sqrt( ( size - 1 ).float ) else 0.0; + + "Equals to sampleError/mean." + shared default Float relativeSampleError => sampleError / mean; + +} diff --git a/source/herd/asynctest/benchmark/StatisticAggregator.ceylon b/source/herd/asynctest/benchmark/StatisticAggregator.ceylon index c4e7b1c..156e122 100644 --- a/source/herd/asynctest/benchmark/StatisticAggregator.ceylon +++ b/source/herd/asynctest/benchmark/StatisticAggregator.ceylon @@ -5,7 +5,7 @@ import java.lang { "Provides thread-unsafe statistic calculations." since( "0.7.0" ) by( "Lis" ) -shared class StatisticAggregator() { +class StatisticAggregator() satisfies Statistic { variable Float minVal = infinity; variable Float maxVal = -infinity; @@ -13,16 +13,10 @@ shared class StatisticAggregator() { variable Float m2Val = 0.0; variable Integer sizeVal = 0; - "Minimum of the values that have been statisticaly treated." - shared Float min => minVal; - "Maximum of the values that have been statisticaly treated." - shared Float max => maxVal; - "Mean calculated within Welford's method of standard deviation computation." - shared Float mean => meanVal; - "Second moment used in Welford's method of standard deviation computation." - shared Float m2 => m2Val; - "The number of the values that have been statisticaly treated." - shared Integer size => sizeVal; + shared actual Float min => minVal; + shared actual Float max => maxVal; + shared actual Float mean => meanVal; + shared actual Integer size => sizeVal; "Resets accumulated statistic to zero state." @@ -36,26 +30,24 @@ shared class StatisticAggregator() { "Returns variance of the values that have been statisticaly treated. The variance is mean((x-mean(x))^2)." - shared Float variance => if ( sizeVal > 1 ) then m2Val / ( sizeVal - 1 ) else 0.0; + shared actual Float variance => if ( sizeVal > 1 ) then m2Val / ( sizeVal - 1 ) else 0.0; "Returns standard deviation of the values that have been statisticaly treated. Standard deviation is `variance^0.5`." - shared Float standardDeviation => Math.sqrt( variance ); + shared actual Float standardDeviation => Math.sqrt( variance ); "Sample variance is size/(size - 1)*variance." - shared Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; + shared actual Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; "Sample standard deviation is sqrt(size/(size - 1)*variance)." - shared Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) * variance ) else 0.0; + shared actual Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) * variance ) else 0.0; "Equals to standardDeviation/sqrt(size)." - shared Float standardError => if ( size > 1 ) then Math.sqrt( variance / size.float ) else 0.0; + shared actual Float standardError => if ( size > 1 ) then Math.sqrt( variance / size.float ) else 0.0; "Equals to sampleStandardDeviation/sqrt(size)." - shared Float sampleError => if ( size > 1 ) then Math.sqrt( variance / ( size - 1 ).float ) else 0.0; - - "Equals to sampleError/mean." - shared Float relativeSampleError => sampleError / mean; + shared actual Float sampleError => if ( size > 1 ) then Math.sqrt( variance / ( size - 1 ).float ) else 0.0; + "Adds sample to the statistic data." shared void sample( Float val ) { diff --git a/source/herd/asynctest/benchmark/StatisticSummary.ceylon b/source/herd/asynctest/benchmark/StatisticSummary.ceylon index 68af0aa..5ff6f7e 100644 --- a/source/herd/asynctest/benchmark/StatisticSummary.ceylon +++ b/source/herd/asynctest/benchmark/StatisticSummary.ceylon @@ -7,19 +7,13 @@ import java.lang { tagged( "Result" ) see( `class ParameterResult` ) since( "0.6.0" ) by( "Lis" ) -shared final class StatisticSummary { +shared final class StatisticSummary satisfies Statistic { - "Minimum of the values that have been statisticaly treated." - shared Float min; - "Maximum of the values that have been statisticaly treated." - shared Float max; - "Mean value." - shared Float mean; - "Returns standard deviation of the values that have been statisticaly treated. - Standard deviation is `variance^0.5`." - shared Float standardDeviation; - "The number of the values that have been statisticaly treated." - shared Integer size; + shared actual Float min; + shared actual Float max; + shared actual Float mean; + shared actual Float standardDeviation; + shared actual Integer size; "Creates new statistic summary with the given data." shared new ( @@ -58,27 +52,7 @@ shared final class StatisticSummary { this.standardDeviation = Math.sqrt( variance ); } - - "Variance of the values that have been statisticaly treated. - The variance is mean((x-mean(x))^2)." - shared Float variance => standardDeviation * standardDeviation; - - "Sample variance is size/(size - 1)*variance." - shared Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; - - "Sample standard deviation is sqrt(size/(size - 1)*variance)." - shared Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) ) * standardDeviation else 0.0; - - "Equals to standardDeviation/sqrt(size)." - shared Float standardError => if ( size > 1 ) then standardDeviation / Math.sqrt( size.float ) else 0.0; - - "Equals to sampleStandardDeviation/sqrt(size)." - shared Float sampleError => if ( size > 1 ) then standardDeviation / Math.sqrt( ( size - 1 ).float ) else 0.0; - - "Equals to sampleError/mean." - shared Float relativeSampleError => sampleError / mean; - - + shared actual Boolean equals( Object other ) { if ( is StatisticSummary other ) { return min == other.min && max == other.max diff --git a/source/herd/asynctest/benchmark/SyncBench.ceylon b/source/herd/asynctest/benchmark/SyncBench.ceylon deleted file mode 100644 index 5d4f78a..0000000 --- a/source/herd/asynctest/benchmark/SyncBench.ceylon +++ /dev/null @@ -1,27 +0,0 @@ -import java.lang { - Runnable -} - - -"Bench with start synchronization and completion notification." -since( "0.7.0" ) by( "Lis" ) -class SyncBench ( - "Bench title. Generally unique." - String title, - "Function to be tested." - shared actual Anything(*Parameter) bench, - "Completed notification." - void completed( StatisticSummary stat ), - "Options the bench has to be executed with." - Options options, - "Parameters the bench has to be executed with." - Parameter parameter -) - extends BaseBench( title ) - satisfies Runnable - given Parameter satisfies Anything[] -{ - - shared actual void run() => completed( execute( options, parameter ) ); - -} diff --git a/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon b/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon deleted file mode 100644 index 1c149dd..0000000 --- a/source/herd/asynctest/benchmark/ThreadLocalClock.ceylon +++ /dev/null @@ -1,85 +0,0 @@ -import java.lang.management { - ManagementFactory -} - - -"Represents clock local to thread - has to be executed from a one thread." -since( "0.7.0" ) by( "Lis" ) -interface ThreadLocalClock { - - "Starts time interval measure from the current time." - shared formal void start(); - - "Measures time interval from the last [[start]] call and up to now." - shared formal Integer measure(); -} - - -"Base for the CPU time interval measurements. - Calculates a factor to scale wall time interval corresponds to measure request. - The factor is moving averaged ratio of CPU time interval to wall time interval. - Ends of CPU time interval are measured as close as possible to ends of wall time interval. - - [[readCurrentTime]] provides particular implementation of CPU or user times. - " -since( "0.7.0" ) by( "Lis" ) -abstract class ThreadClock() satisfies ThreadLocalClock { - - variable Integer startWallTime = system.nanoseconds; - variable Integer lastWallTime = 0; - variable Integer lastSample = 0; - variable Float factor = 1.0; - - if ( !ManagementFactory.threadMXBean.threadCpuTimeEnabled ) { - ManagementFactory.threadMXBean.threadCpuTimeEnabled = true; - } - - "Returns current time based on strategy." - shared formal Integer readCurrentTime(); - - - shared actual void start() { - Integer currentSample = readCurrentTime(); - Integer currentWallTime = system.nanoseconds; - if ( lastSample > 0 && currentSample > lastSample && currentWallTime > lastWallTime ) { - // Moving averaging factor! - factor = 0.9 * factor + 0.1 * ( currentSample - lastSample ) / ( currentWallTime - lastWallTime ); - lastSample = currentSample; - lastWallTime = currentWallTime; - } - startWallTime = system.nanoseconds; - } - - shared actual Integer measure() => ( factor * ( system.nanoseconds - startWallTime ) ).integer; - -} - - -"CPU clock uses `ThreadMXBean.currentThreadCpuTime` to measure time interval." -since( "0.7.0" ) by( "Lis" ) -class CPULocalClock() extends ThreadClock() { - shared actual Integer readCurrentTime() { - value threadMXBean = ManagementFactory.threadMXBean; - if ( threadMXBean.currentThreadCpuTimeSupported ) { - return threadMXBean.currentThreadCpuTime; - } - else { - return system.nanoseconds; - } - } -} - - -"User clock uses `ThreadMXBean.currentThreadUserTime` to measure time interval." -since( "0.7.0" ) by( "Lis" ) -class UserLocalClock() extends ThreadClock() { - shared actual Integer readCurrentTime() { - value threadMXBean = ManagementFactory.threadMXBean; - if ( threadMXBean.currentThreadCpuTimeSupported ) { - return threadMXBean.currentThreadUserTime; - } - else { - return system.nanoseconds; - } - } -} diff --git a/source/herd/asynctest/benchmark/ThreadableRunner.ceylon b/source/herd/asynctest/benchmark/ThreadableRunner.ceylon new file mode 100644 index 0000000..010df0b --- /dev/null +++ b/source/herd/asynctest/benchmark/ThreadableRunner.ceylon @@ -0,0 +1,135 @@ +import java.lang { + Runnable, + Thread, + System +} +import java.util.concurrent { + CyclicBarrier +} +import java.util.concurrent.atomic { + AtomicInteger +} + + +"Base class for family of benches runners in several threads." +since( "0.7.0" ) by( "Lis" ) +abstract class ThreadableRunner( "Options the bench is executed with." Options options ) { + + "Barrier which synchronizes loop starting." + variable CyclicBarrier loopBarrier = CyclicBarrier( 1 ); + "Barrier which synchronizes statistic calculation." + variable CyclicBarrier calculateBarrier = CyclicBarrier( 1 ); + "`true` if benchmarkis completed and `false` otherwise." + variable Boolean running = true; + "Total number of threads currently invoking bench function." + AtomicInteger benchThreads = AtomicInteger( 0 ); + + + shared class RunnableBench( "Benchmark function." BenchFlow<[]> bench ) satisfies Runnable { + + "Time spent in the latest loop." + shared variable Float loopTime = 0.0; + + "Level of concurency is mean number of threads invoked bench functions concurrently in the latest loop." + shared variable Float concurrencyLevel = 0.0; + + shared actual void run() { + Clock clock = options.clock(); + bench.setup(); + while ( running ) { + // test loop + variable Integer concurrentSamples = 0; + concurrencyLevel = 0.0; + loopTime = 0.0; + variable Integer loop = options.iterationsPerLoop; + while ( loop -- > 0 ) { + bench.before(); + // number of GC starts before test run + Integer numGCBefore = numberOfGCRuns(); + // execute the test function + clock.start(); + Integer threadsBefore = benchThreads.incrementAndGet(); + Anything ret = bench.bench(); + Integer threadsAfter = benchThreads.andDecrement; + Float delta = clock.measure( options.timeUnit ); + // number of GC starts after test run + Integer numGCAfter = numberOfGCRuns(); + if ( numGCAfter == numGCBefore || !options.skipGCRuns ) { + // time spent on bench function execution + loopTime += delta; + // mean number of concurrently bench function threads + concurrentSamples ++; + Float concurrentDelta = 0.5 * ( threadsBefore + threadsAfter ) - concurrencyLevel; + concurrencyLevel = concurrencyLevel + concurrentDelta / concurrentSamples; + } + else { + loop ++; + } + // Eleminate JIT optimization + pushToBlackHole( ret ); + bench.after(); + } + // await while all loops to be completed + loopBarrier.await(); + // await statistic data calculations to be completed + calculateBarrier.await(); + } + bench.dispose(); + } + } + + + "Instantiates benches." + shared formal RunnableBench[] benches; + + + "Executes the benchmark." + shared StatisticSummary execute() { + running = true; + benchThreads.set( 0 ); + StatisticAggregator calculator = StatisticAggregator(); + loopBarrier = CyclicBarrier( benches.size + 1 ); + calculateBarrier = CyclicBarrier( benches.size + 1 ); + variable CompletionCriterion? warmupCriterion = options.warmupCriterion; + Integer totalBenches = benches.size; + + // start benches + for ( item in benches ) { + Thread( item ).start(); + } + // accumulate stat + while ( true ) { + loopBarrier.await(); + // calculate operations per time units + variable Float meanOperations = 0.0; + variable Float meanConcurrencyLevel = 0.0; + for ( item in benches ) { + meanOperations += options.iterationsPerLoop / item.loopTime; + meanConcurrencyLevel += item.concurrencyLevel; + } + meanConcurrencyLevel /= totalBenches; + meanOperations /= totalBenches; + calculator.sample( meanConcurrencyLevel * meanOperations ); + + // completion verifying + if ( exists criterion = warmupCriterion ) { + if ( criterion.verify( calculator, options.timeUnit ) ) { + // warmup round is completed + warmupCriterion = null; + calculator.reset(); + System.gc(); + } + } + else { + if ( options.measureCriterion.verify( calculator, options.timeUnit ) ) { + // measure round is completed + running = false; + calculateBarrier.await(); + return calculator.result; + } + } + benchThreads.set( 0 ); + calculateBarrier.await(); + } + } +} diff --git a/source/herd/asynctest/benchmark/TimeCriteria.ceylon b/source/herd/asynctest/benchmark/TimeCriteria.ceylon deleted file mode 100644 index fde3c3e..0000000 --- a/source/herd/asynctest/benchmark/TimeCriteria.ceylon +++ /dev/null @@ -1,125 +0,0 @@ -import java.util.concurrent.atomic { - AtomicLong -} -import java.lang { - Thread -} -import java.util.concurrent { - ConcurrentSkipListMap -} - - -"Continues benchmark iterations while total execution time doesn't exceed [[timeLimit]]. - Execution time is overall time spent on test execution. - I.e. it summarizes time spent on benchmark function execution as well as on internal calculations. - - The alternative way is take into account time spent on benchmark function execution only is realized - in [[LocalBenchTime]] and [[TotalBenchTime]]. - " -tagged( "Criteria" ) -see( `class Options`, `class LocalBenchTime`, `class TotalBenchTime` ) -throws( `class AssertionError`, "Maximum permited execution is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class TotalExecutionTime ( - "Maximum permited execution time in [[timeUnit]] units." Integer timeLimit, - "Unit the [[timeLimit]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds -) - satisfies CompletionCriterion -{ - "Maximum permited execution has to be > 0." - assert( timeLimit > 0 ); - Integer totalNanoseconds = ( timeLimit * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; - - variable Integer startTime = 0; - - - shared actual void reset() { - startTime = system.nanoseconds; - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit deltaTimeUnit ) { - return system.nanoseconds - startTime > totalNanoseconds; - } - -} - - -"Continues benchmark iterations while overall time accumulated by benchmark iterations doesn't exceed - [[totalTime]]. - This criterion summarizes only time spent on benchmark function execution for all execution threads jointly. - - If benchmark function execution time is to be summarized for each thread separately [[LocalBenchTime]] is to be used. - If overall execution time has to be taken into account [[TotalExecutionTime]] is to be used. - " -tagged( "Criteria" ) -see( `class Options`, `class LocalBenchTime`, `class TotalExecutionTime` ) -throws( `class AssertionError`, "Total benchmark time is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class TotalBenchTime ( - "Maximum time (in [[timeUnit]] units) to be accumulated by benchmark iterations." Integer totalTime, - "Unit the [[totalTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds -) - satisfies CompletionCriterion -{ - "Total benchmark time has to be > 0." - assert( totalTime > 0 ); - Integer totalNanoseconds = ( totalTime * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; - - AtomicLong accumulatedTime = AtomicLong( 0 ); - - - shared actual void reset() { - accumulatedTime.set( 0 ); - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit deltaTimeUnit ) { - Integer timeToAdd = ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; - return accumulatedTime.addAndGet( timeToAdd ) > totalNanoseconds; - } - -} - - -"Continues benchmark iterations while local (relative to thread) time accumulated by benchmark iterations doesn't exceed - [[localTime]]. - This criterion summarizes time spent on benchmark function execution for each thread separately. - - If benchmark function execution time is to be summarized for all threads [[TotalBenchTime]] is to be used. - If overall execution time has to be taken into account [[TotalExecutionTime]] is to be used. - " -tagged( "Criteria" ) -see( `class Options`, `class TotalBenchTime`, `class TotalExecutionTime` ) -throws( `class AssertionError`, "Local benchmark time is <= 0." ) -since( "0.7.0" ) by( "Lis" ) -shared class LocalBenchTime ( - "Maximum time (in [[timeUnit]] units) to be accumulated by local (relative to thread) iterations." Integer localTime, - "Unit the [[localTime]] is measured in. Default is milliseconds." TimeUnit timeUnit = TimeUnit.milliseconds -) - satisfies CompletionCriterion -{ - - "Thread local benchmark time has to be > 0." - assert( localTime > 0 ); - Integer nanoseconds = ( localTime * timeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; - - ConcurrentSkipListMap localAccumulators = ConcurrentSkipListMap( IntegerComparator() ); - - - shared actual void reset() { - localAccumulators.clear(); - } - - shared actual Boolean verify( Float delta, StatisticAggregator stat, TimeUnit deltaTimeUnit ) { - Integer id = Thread.currentThread().id; - Integer timeVal; - if ( exists current = localAccumulators.get( id ) ) { - timeVal = current + ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; - } - else { - timeVal = ( delta * deltaTimeUnit.factorToSeconds / TimeUnit.nanoseconds.factorToSeconds + 0.5 ).integer; - } - localAccumulators.put( id, timeVal ); - return timeVal > nanoseconds; - } - -} diff --git a/source/herd/asynctest/benchmark/benchmark.ceylon b/source/herd/asynctest/benchmark/benchmark.ceylon index fc78f08..9f8b18f 100644 --- a/source/herd/asynctest/benchmark/benchmark.ceylon +++ b/source/herd/asynctest/benchmark/benchmark.ceylon @@ -1,5 +1,7 @@ import ceylon.collection { - HashMap + HashMap, + Hashtable, + linked } import java.lang { Thread, @@ -10,6 +12,8 @@ import java.lang { "Clears black hole and resets criteria." since( "0.7.0" ) by( "Lis" ) void initializeBench( Options options ) { + Thread.sleep( 50 ); + System.gc(); blackHole.clear(); options.measureCriterion.reset(); options.warmupCriterion?.reset(); @@ -24,20 +28,10 @@ void completeBench( Options options ) { blackHole.verifyNumbers(); } -"Runs garbage collector and sleeps for a short amount of time." -since( "0.7.0" ) by( "Lis" ) -void warmupBenchmark() { - System.gc(); - Thread.sleep( 200 ); - System.gc(); - Thread.sleep( 200 ); -} - "Runs benchmark testing. Executes each given bench with each given parameter. Bench is responsible for the execution details and calculation performance statistic." -throws( `class AssertionError`, "number of measure rounds is <= 0" ) since( "0.7.0" ) by( "Lis" ) shared Result benchmark ( "Benchmark options of benches executing." @@ -49,15 +43,16 @@ shared Result benchmark ( ) given Parameter satisfies Anything[] { - warmupBenchmark(); - if ( nonempty params = parameters ) { - // run test for each parameter and each bench - HashMap> res = HashMap>(); + HashMap> res = HashMap> ( + linked, Hashtable( params.size ) + ); for ( param in params ) { if ( !res.defines( param ) ) { - HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary>(); + HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary> ( + linked, Hashtable( benches.size ) + ); for ( bench in benches ) { if ( !benchRes.defines( bench ) ) { // initialize criteria and black hole @@ -83,7 +78,9 @@ shared Result benchmark ( assert ( is [Bench<[]>+] benches ); HashMap<[], ParameterResult<[]>> res = HashMap<[], ParameterResult<[]>>(); - HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary>(); + HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary> ( + linked, Hashtable( benches.size ) + ); for ( bench in benches ) { if ( !benchRes.defines( bench ) ) { // initialize criteria and black hole diff --git a/source/herd/asynctest/benchmark/contextWriters.ceylon b/source/herd/asynctest/benchmark/contextWriters.ceylon index f6dcb85..e13427c 100644 --- a/source/herd/asynctest/benchmark/contextWriters.ceylon +++ b/source/herd/asynctest/benchmark/contextWriters.ceylon @@ -32,7 +32,7 @@ shared void writeAbsolute( AsyncTestContext context, Result( AsyncTestContext context, Result( AsyncTestContext context, Result< + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to slowest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " - + "runs = ``stat.size``" + + "loops = ``stat.size``" ); } } @@ -80,7 +80,7 @@ shared void writeRelativeToSlowest( AsyncTestContext context, Result< + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to slowest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " - + "runs = ``stat.size``" + + "loops = ``stat.size``" ); } } @@ -107,7 +107,7 @@ shared void writeRelativeToFastest( AsyncTestContext context, Result< + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to fastest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " - + "runs = ``stat.size``" + + "loops = ``stat.size``" ); } } @@ -118,7 +118,7 @@ shared void writeRelativeToFastest( AsyncTestContext context, Result< + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " + "``stat.relativeMean``% to fastest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " - + "runs = ``stat.size``" + + "loops = ``stat.size``" ); } } diff --git a/source/herd/asynctest/benchmark/numberOfGCRuns.ceylon b/source/herd/asynctest/benchmark/numberOfGCRuns.ceylon new file mode 100644 index 0000000..870f9ec --- /dev/null +++ b/source/herd/asynctest/benchmark/numberOfGCRuns.ceylon @@ -0,0 +1,11 @@ +import java.lang.management { + ManagementFactory +} + + +"Returns number of GC runs up to now." +Integer numberOfGCRuns() { + variable Integer numGC = 0; + for ( gcBean in ManagementFactory.garbageCollectorMXBeans ) { numGC += gcBean.collectionCount; } + return numGC; +} diff --git a/source/herd/asynctest/internal/StatisticCalculator.ceylon b/source/herd/asynctest/internal/StatisticCalculator.ceylon index d516afc..d05eb6a 100644 --- a/source/herd/asynctest/internal/StatisticCalculator.ceylon +++ b/source/herd/asynctest/internal/StatisticCalculator.ceylon @@ -128,6 +128,12 @@ shared class StatisticCalculator() { } } + "Mean calculated within Welford's method of standard deviation computation." + shared Float mean => stat.get().mean; + "The number of the values that have been statisticaly treated." + shared Integer size => stat.get().size; + + "Statistic summary accumulated up to the query moment." see( `function sample`, `function samples` ) shared StatisticSummary statisticSummary => stat.get().summary; From 45c58eae366185264e89efb041e9173d4f986ac2 Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 14 Mar 2017 07:01:42 +0300 Subject: [PATCH 62/72] CloseTo matcher tagged with numbers --- .../asynctest/match/comparableMatchers.ceylon | 19 ------------------ .../asynctest/match/numberMatchers.ceylon | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/source/herd/asynctest/match/comparableMatchers.ceylon b/source/herd/asynctest/match/comparableMatchers.ceylon index ba3ca3a..e00bb46 100644 --- a/source/herd/asynctest/match/comparableMatchers.ceylon +++ b/source/herd/asynctest/match/comparableMatchers.ceylon @@ -167,22 +167,3 @@ shared class NotInRange ( return "range <``typeName( tVal )``>"; } } - - -"Verifies if matching value is close to `merit` with the given `tolerance`." -tagged( "Comparators" ) since( "0.4.0" ) by( "Lis" ) -shared class CloseTo ( - "Value to compare with matching one." Value merit, - "Tolerance to accept matching." Value tolerance -) - satisfies Matcher - given Value satisfies Comparable & Number -{ - shared actual MatchResult match( Value val ) - => MatchResult ( - "``stringify( val )`` is close to ``stringify( merit )`` with tolerance of ``tolerance``", - ( val - merit ).magnitude < tolerance - ); - - shared actual String string => "close with tolerance ``tolerance``"; -} diff --git a/source/herd/asynctest/match/numberMatchers.ceylon b/source/herd/asynctest/match/numberMatchers.ceylon index 252df37..e4eb4df 100644 --- a/source/herd/asynctest/match/numberMatchers.ceylon +++ b/source/herd/asynctest/match/numberMatchers.ceylon @@ -159,3 +159,23 @@ shared class IsOdd() satisfies Matcher shared actual String string => "is odd"; } + + +"Verifies if matching value is close to `merit` with the given `tolerance`." +tagged( "Numbers" ) see( `class EqualTo` ) +since( "0.4.0" ) by( "Lis" ) +shared class CloseTo ( + "Value to compare with matching one." Value merit, + "Tolerance to accept matching." Value tolerance +) + satisfies Matcher + given Value satisfies Number + { + shared actual MatchResult match( Value val ) + => MatchResult ( + "``stringify( val )`` is close to ``stringify( merit )`` with tolerance of ``tolerance``", + ( val - merit ).magnitude < tolerance + ); + + shared actual String string => "close with tolerance ``tolerance``"; +} From 7ab6d93b31cb94518b46dc3bfc5fdc452ff09312 Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 14 Mar 2017 07:03:16 +0300 Subject: [PATCH 63/72] Statistic is interface now --- .../benchmark/ComparativeStatistic.ceylon | 71 ------------------- .../herd/asynctest/benchmark/Statistic.ceylon | 4 ++ .../StatisticAggregator.ceylon | 7 +- .../internal/StatisticCalculator.ceylon | 12 ++-- .../StatisticSummary.ceylon | 4 +- source/herd/asynctest/rule/MeterRule.ceylon | 8 +-- .../herd/asynctest/rule/StatisticRule.ceylon | 6 +- 7 files changed, 27 insertions(+), 85 deletions(-) delete mode 100644 source/herd/asynctest/benchmark/ComparativeStatistic.ceylon rename source/herd/asynctest/{benchmark => internal}/StatisticAggregator.ceylon (93%) rename source/herd/asynctest/{benchmark => internal}/StatisticSummary.ceylon (98%) diff --git a/source/herd/asynctest/benchmark/ComparativeStatistic.ceylon b/source/herd/asynctest/benchmark/ComparativeStatistic.ceylon deleted file mode 100644 index 2cfa627..0000000 --- a/source/herd/asynctest/benchmark/ComparativeStatistic.ceylon +++ /dev/null @@ -1,71 +0,0 @@ -import java.lang { - Math -} - - -"Represents statistic values comparing to another [[StatisticSummary]]." -tagged( "Result" ) -see( `class StatisticSummary` ) -since( "0.7.0" ) by( "Lis" ) -shared final class ComparativeStatistic ( - "Statistic to be compared to [[baseMean]]." shared StatisticSummary stat, - "Base mean value the [[stat]] is compared to." shared Float baseMean -) { - - "Minimum of the values that have been statisticaly treated." - shared Float min => stat.min; - - "Maximum of the values that have been statisticaly treated." - shared Float max => stat.max; - - "Mean value." - shared Float mean => stat.mean; - - "Returns standard deviation of the values that have been statisticaly treated. - Standard deviation is `variance^0.5`." - shared Float standardDeviation => stat.standardDeviation; - - "The number of the values that have been statisticaly treated." - shared Integer size => stat.size; - - "Variance of the values that have been statisticaly treated. - The variance is mean((x-mean(x))^2)." - shared Float variance => stat.variance; - - "Sample variance is size/(size - 1)*variance." - shared Float sampleVariance => if ( size > 1 ) then size * variance / ( size - 1 ) else 0.0; - - "Sample standard deviation is sqrt(size/(size - 1)*variance)." - shared Float sampleDeviation => if ( size > 1 ) then Math.sqrt( size.float / ( size - 1 ) ) * standardDeviation else 0.0; - - "Equals to standardDeviation/sqrt(size)." - shared Float standardError => if ( size > 1 ) then standardDeviation / Math.sqrt( size.float ) else 0.0; - - "Equals to sampleStandardDeviation/sqrt(size)." - shared Float sampleError => if ( size > 1 ) then standardDeviation / Math.sqrt( ( size - 1 ).float ) else 0.0; - - "Equals to sampleError/mean." - shared Float relativeSampleError => sampleError / mean; - - - "Mean value relative to `baseMean` in percents." - shared Integer relativeMean => if ( mean == baseMean ) then 100 else ( mean / baseMean * 100 ).integer; - - - shared actual Boolean equals( Object other ) { - if ( is ComparativeStatistic other ) { - return stat == other.stat && baseMean == other.baseMean; - } - else { - return false; - } - } - - shared actual Integer hash => stat.hash * 37 + baseMean.integer; - - - shared actual String string => "mean=``Float.format(mean, 0, 3)``, standard deviation=``Float.format(standardDeviation, 0, 3)``, " + - "max=``Float.format(max, 0, 3)``, min=``Float.format(min, 0, 3)``, total samples=``size``,"+ - "relative mean=``relativeMean``%, base mean=``Float.format(baseMean, 0, 3)``"; - -} diff --git a/source/herd/asynctest/benchmark/Statistic.ceylon b/source/herd/asynctest/benchmark/Statistic.ceylon index 4a2b240..d1b1e54 100644 --- a/source/herd/asynctest/benchmark/Statistic.ceylon +++ b/source/herd/asynctest/benchmark/Statistic.ceylon @@ -1,6 +1,10 @@ import java.lang { Math } +import herd.asynctest.internal { + StatisticSummary, + StatisticAggregator +} "Provides statistic data for a stream of variate values." diff --git a/source/herd/asynctest/benchmark/StatisticAggregator.ceylon b/source/herd/asynctest/internal/StatisticAggregator.ceylon similarity index 93% rename from source/herd/asynctest/benchmark/StatisticAggregator.ceylon rename to source/herd/asynctest/internal/StatisticAggregator.ceylon index 156e122..c06487e 100644 --- a/source/herd/asynctest/benchmark/StatisticAggregator.ceylon +++ b/source/herd/asynctest/internal/StatisticAggregator.ceylon @@ -2,10 +2,15 @@ import java.lang { Math } +import herd.asynctest.benchmark { + Statistic +} + "Provides thread-unsafe statistic calculations." +see( `class StatisticCalculator` ) since( "0.7.0" ) by( "Lis" ) -class StatisticAggregator() satisfies Statistic { +shared class StatisticAggregator() satisfies Statistic { variable Float minVal = infinity; variable Float maxVal = -infinity; diff --git a/source/herd/asynctest/internal/StatisticCalculator.ceylon b/source/herd/asynctest/internal/StatisticCalculator.ceylon index d05eb6a..d75c422 100644 --- a/source/herd/asynctest/internal/StatisticCalculator.ceylon +++ b/source/herd/asynctest/internal/StatisticCalculator.ceylon @@ -1,12 +1,13 @@ import java.util.concurrent.atomic { AtomicReference } -import herd.asynctest.benchmark { - StatisticSummary -} + import java.lang { Math } +import herd.asynctest.benchmark { + Statistic +} "Current values of statistic calculations." @@ -112,7 +113,8 @@ class StatisticStream { } -"Calculates statistic data for stream of variate values." +"Provides thread-safe statistic calculations." +see( `class StatisticAggregator` ) since( "0.6.0" ) by( "Lis" ) shared class StatisticCalculator() { @@ -136,7 +138,7 @@ shared class StatisticCalculator() { "Statistic summary accumulated up to the query moment." see( `function sample`, `function samples` ) - shared StatisticSummary statisticSummary => stat.get().summary; + shared Statistic statisticSummary => stat.get().summary; "Thread-safely adds a one sample to the statistic." see( `value statisticSummary`, `function samples` ) diff --git a/source/herd/asynctest/benchmark/StatisticSummary.ceylon b/source/herd/asynctest/internal/StatisticSummary.ceylon similarity index 98% rename from source/herd/asynctest/benchmark/StatisticSummary.ceylon rename to source/herd/asynctest/internal/StatisticSummary.ceylon index 5ff6f7e..b14c67a 100644 --- a/source/herd/asynctest/benchmark/StatisticSummary.ceylon +++ b/source/herd/asynctest/internal/StatisticSummary.ceylon @@ -1,11 +1,13 @@ import java.lang { Math } +import herd.asynctest.benchmark { + Statistic +} "Statistic summary for a stream of variate values." tagged( "Result" ) -see( `class ParameterResult` ) since( "0.6.0" ) by( "Lis" ) shared final class StatisticSummary satisfies Statistic { diff --git a/source/herd/asynctest/rule/MeterRule.ceylon b/source/herd/asynctest/rule/MeterRule.ceylon index da98e92..304451e 100644 --- a/source/herd/asynctest/rule/MeterRule.ceylon +++ b/source/herd/asynctest/rule/MeterRule.ceylon @@ -5,7 +5,7 @@ import java.util.concurrent.atomic { AtomicLong } import herd.asynctest.benchmark { - StatisticSummary + Statistic } import herd.asynctest.internal { StatisticCalculator @@ -33,7 +33,7 @@ import herd.asynctest.internal { } " -see( `class StatisticSummary` ) +see( `interface Statistic` ) since( "0.6.0" ) by( "Lis" ) tagged( "TestRule" ) shared class MeterRule() satisfies TestRule { @@ -53,10 +53,10 @@ tagged( "TestRule" ) shared class MeterRule() satisfies TestRule "Statistic summary for execution time." - shared StatisticSummary timeStatistic => store.element.timeCalculator.statisticSummary; + shared Statistic timeStatistic => store.element.timeCalculator.statisticSummary; "Statistic summary for rate (operations per second)." - shared StatisticSummary rateStatistic => store.element.rateCalculator.statisticSummary; + shared Statistic rateStatistic => store.element.rateCalculator.statisticSummary; "Starts metering from now and memoizes current system time. diff --git a/source/herd/asynctest/rule/StatisticRule.ceylon b/source/herd/asynctest/rule/StatisticRule.ceylon index 2dfcf8c..a3cb780 100644 --- a/source/herd/asynctest/rule/StatisticRule.ceylon +++ b/source/herd/asynctest/rule/StatisticRule.ceylon @@ -2,7 +2,7 @@ import herd.asynctest { AsyncPrePostContext } import herd.asynctest.benchmark { - StatisticSummary + Statistic } import herd.asynctest.internal { StatisticCalculator @@ -12,7 +12,7 @@ import herd.asynctest.internal { "Lock-free and thread-safely accumulates statistics data of some variate values. Doesn't collect values, just accumulates statistic data when sample added - see [[sample]] and [[samples]]. Statistic data is reseted before _each_ test. " -see( `class StatisticSummary` ) +see( `interface Statistic` ) tagged( "TestRule" ) since( "0.6.0" ) by( "Lis" ) shared class StatisticRule() satisfies TestRule { @@ -23,7 +23,7 @@ shared class StatisticRule() satisfies TestRule "Statistic summary accumulated up to the query moment." see( `function samples`, `function sample` ) - shared StatisticSummary statisticSummary => calculator.element.statisticSummary; + shared Statistic statisticSummary => calculator.element.statisticSummary; "Thread-safely adds a one sample to the statistic." see( `value statisticSummary`, `function samples` ) From 8ff4e30379e606e34fcf1d94ad86732a836582d7 Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 14 Mar 2017 07:04:58 +0300 Subject: [PATCH 64/72] doc --- source/herd/asynctest/AsyncTestContext.ceylon | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/herd/asynctest/AsyncTestContext.ceylon b/source/herd/asynctest/AsyncTestContext.ceylon index eae8eaa..6a8f473 100644 --- a/source/herd/asynctest/AsyncTestContext.ceylon +++ b/source/herd/asynctest/AsyncTestContext.ceylon @@ -34,7 +34,7 @@ import herd.asynctest.match { " see( `class AsyncTestExecutor`, `package herd.asynctest.match` ) since( "0.0.1" ) by( "Lis" ) -shared interface AsyncTestContext// satisfies AsyncMessageContext +shared interface AsyncTestContext { "Completes the testing. To be called by the test function when testing is completed. @@ -98,7 +98,8 @@ shared interface AsyncTestContext// satisfies AsyncMessageContext "Optional title to be shown within test name." String title = "", "If `true` reports on failures and successes. - Otherwise reportes only on failures." + Otherwise reportes only on failures. + Default is `false`." Boolean reportSuccess = false ) { try { From 37ad50ed5311cb98534da5b1a6b2ca078ed53a12 Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 21 Mar 2017 09:15:30 +0300 Subject: [PATCH 65/72] bench flows --- .../benchmark/LockFreeQueueBenchmark.ceylon | 2 +- .../benchmark/RandomTestFunction.ceylon | 50 +++++++++++++ .../herd/examples/asynctest/module.ceylon | 20 +++--- .../benchmark/AbstractSelectiveFlow.ceylon | 71 +++++++++++++++++++ source/herd/asynctest/benchmark/Bench.ceylon | 2 +- .../herd/asynctest/benchmark/BenchFlow.ceylon | 13 ++-- .../asynctest/benchmark/BenchResult.ceylon | 66 +++++++++++++++++ .../herd/asynctest/benchmark/Criteria.ceylon | 2 +- .../asynctest/benchmark/MultiBench.ceylon | 2 +- .../herd/asynctest/benchmark/Options.ceylon | 4 ++ .../benchmark/ParameterResult.ceylon | 49 ++++--------- .../asynctest/benchmark/RandomDataFlow.ceylon | 40 +++++++++++ .../asynctest/benchmark/RandomFlow.ceylon | 47 ++++++++++++ source/herd/asynctest/benchmark/Result.ceylon | 14 ++-- .../asynctest/benchmark/SelectiveFlow.ceylon | 15 ++++ .../asynctest/benchmark/SequentialFlow.ceylon | 39 ++++++++++ .../asynctest/benchmark/SingleBench.ceylon | 20 +++++- .../benchmark/ThreadableRunner.ceylon | 26 ++++++- .../herd/asynctest/benchmark/benchmark.ceylon | 4 +- .../asynctest/benchmark/contextWriters.ceylon | 22 +++--- .../herd/asynctest/benchmark/package.ceylon | 18 ++++- .../benchmark/reducedProbability.ceylon | 31 ++++++++ source/herd/asynctest/module.ceylon | 6 +- 23 files changed, 480 insertions(+), 83 deletions(-) create mode 100644 examples/herd/examples/asynctest/benchmark/RandomTestFunction.ceylon create mode 100644 source/herd/asynctest/benchmark/AbstractSelectiveFlow.ceylon create mode 100644 source/herd/asynctest/benchmark/BenchResult.ceylon create mode 100644 source/herd/asynctest/benchmark/RandomDataFlow.ceylon create mode 100644 source/herd/asynctest/benchmark/RandomFlow.ceylon create mode 100644 source/herd/asynctest/benchmark/SelectiveFlow.ceylon create mode 100644 source/herd/asynctest/benchmark/SequentialFlow.ceylon create mode 100644 source/herd/asynctest/benchmark/reducedProbability.ceylon diff --git a/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon b/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon index cf043ac..2fddcc4 100644 --- a/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon +++ b/examples/herd/examples/asynctest/benchmark/LockFreeQueueBenchmark.ceylon @@ -28,7 +28,7 @@ shared test async void lockFreeQueueProducerConsumer(AsyncTestContext context) { writeRelativeToFastest ( context, benchmark ( - Options(NumberOfLoops(500), NumberOfLoops(100), 100, TimeUnit.milliseconds, CPUClock, true), + Options(NumberOfLoops(500), NumberOfLoops(100), 100, TimeUnit.milliseconds, CPUClock), [ MultiBench( "producer-consumer", diff --git a/examples/herd/examples/asynctest/benchmark/RandomTestFunction.ceylon b/examples/herd/examples/asynctest/benchmark/RandomTestFunction.ceylon new file mode 100644 index 0000000..614f1f9 --- /dev/null +++ b/examples/herd/examples/asynctest/benchmark/RandomTestFunction.ceylon @@ -0,0 +1,50 @@ +import herd.asynctest { + async, + AsyncTestContext +} +import herd.asynctest.benchmark { + NumberOfLoops, + benchmark, + Options, + RandomFlow, + SingleBench +} +import ceylon.test { + test +} +import herd.asynctest.match { + CloseTo +} + + +shared class RandomTestFunction() { + + + variable Integer f1Calls = 0; + variable Integer f2Calls = 0; + variable Integer f3Calls = 0; + + void f1() => f1Calls ++; + void f2() => f2Calls ++; + void f3() => f3Calls ++; + + + shared async test void selectFunctionRandomly(AsyncTestContext context) { + benchmark ( + Options(NumberOfLoops(1000), NumberOfLoops(100)), + [ + SingleBench ( + "random test function", + RandomFlow(1, [f1, 0.3], [f2, 0.2], [f3, 0.5]) + ) + ] + ); + Integer sum = f1Calls + f2Calls + f3Calls; + context.assertThat((f1Calls.float/sum*100+0.5).integer, CloseTo(30, 2), "first function % of calls", true); + context.assertThat((f2Calls.float/sum*100+0.5).integer, CloseTo(20, 2), "second function % of calls", true); + context.assertThat((f3Calls.float/sum*100+0.5).integer, CloseTo(50, 2), "third function % of calls", true); + context.complete(); + } + +} + diff --git a/examples/herd/examples/asynctest/module.ceylon b/examples/herd/examples/asynctest/module.ceylon index c3beb89..f72e0a6 100644 --- a/examples/herd/examples/asynctest/module.ceylon +++ b/examples/herd/examples/asynctest/module.ceylon @@ -33,16 +33,16 @@ license ( by( "Lis" ) native( "jvm" ) module herd.examples.asynctest "0.7.0" { - shared import ceylon.collection "1.3.1"; - shared import ceylon.http.server "1.3.1"; - shared import ceylon.http.client "1.3.1"; - shared import ceylon.buffer "1.3.1"; - shared import ceylon.http.common "1.3.1"; - shared import ceylon.json "1.3.1"; - shared import ceylon.file "1.3.1"; - shared import ceylon.promise "1.3.1"; - shared import ceylon.test "1.3.1"; + shared import ceylon.collection "1.3.2"; + shared import ceylon.http.server "1.3.2"; + shared import ceylon.http.client "1.3.2"; + shared import ceylon.buffer "1.3.2"; + shared import ceylon.http.common "1.3.2"; + shared import ceylon.json "1.3.2"; + shared import ceylon.file "1.3.2"; + shared import ceylon.promise "1.3.2"; + shared import ceylon.test "1.3.2"; shared import herd.asynctest "0.7.0"; shared import java.base "8"; - import ceylon.interop.java "1.3.1"; + import ceylon.interop.java "1.3.2"; } diff --git a/source/herd/asynctest/benchmark/AbstractSelectiveFlow.ceylon b/source/herd/asynctest/benchmark/AbstractSelectiveFlow.ceylon new file mode 100644 index 0000000..8b5987e --- /dev/null +++ b/source/herd/asynctest/benchmark/AbstractSelectiveFlow.ceylon @@ -0,0 +1,71 @@ +import java.lang { + ThreadLocal +} +import java.util.\ifunction { + Supplier +} + + +"Abstract base class for flow which selects another bench function each given number of benchmark iterations. + Bench functions are selected independently for each execution thread." +tagged( "Bench flow" ) +since( "0.7.0" ) by( "Lis" ) +shared abstract class AbstractSelectiveFlow ( +) + satisfies BenchFlow + given Parameter satisfies Anything[] +{ + + ThreadLocal localIterations = ThreadLocal.withInitial ( + object satisfies Supplier { + shared actual Integer get() => 0; + } + ); + ThreadLocal numOfIterations = ThreadLocal.withInitial ( + object satisfies Supplier { + shared actual Integer get() => 0; + } + ); + ThreadLocal currentBench = ThreadLocal(); + + void setNumOfIterations() { + value it = iterations; + if ( is Integer it ) { + numOfIterations.set( it ); + } + else { + numOfIterations.set( it() ); + } + } + + + "Selects another bench function each time when called." + shared formal Anything(*Parameter) select(); + + "Source of a number of iterations after which bench function has to be reselected. + Might be variable. + If <= 0 bench function is reselected at each iteration." + shared formal Integer | Integer() iterations; + + + shared actual Anything(*Parameter) bench => currentBench.get(); + + shared actual default void setup() { + currentBench.set( select() ); + setNumOfIterations(); + } + shared actual default void before() {} + shared actual default void after() { + Integer current = localIterations.get(); + if ( current >= numOfIterations.get() ) { + setNumOfIterations(); + currentBench.set( select() ); + localIterations.set( 0 ); + } + else { + localIterations.set( current + 1 ); + } + } + shared actual default void dispose() {} + +} diff --git a/source/herd/asynctest/benchmark/Bench.ceylon b/source/herd/asynctest/benchmark/Bench.ceylon index 5cd2c30..21231fd 100644 --- a/source/herd/asynctest/benchmark/Bench.ceylon +++ b/source/herd/asynctest/benchmark/Bench.ceylon @@ -13,7 +13,7 @@ shared interface Bench "Executes this bench with the given `Parameter` value. Returns statistic of operations per second for the execution." - shared formal StatisticSummary execute ( + shared formal BenchResult execute ( "Options of the bench execution." Options options, "Value of the parameter the bench is parameterized with." Parameter parameter ); diff --git a/source/herd/asynctest/benchmark/BenchFlow.ceylon b/source/herd/asynctest/benchmark/BenchFlow.ceylon index e270dfd..20f6ae5 100644 --- a/source/herd/asynctest/benchmark/BenchFlow.ceylon +++ b/source/herd/asynctest/benchmark/BenchFlow.ceylon @@ -1,8 +1,8 @@ "Represents benchmark function + test flow. I.e. functions called before / after test iteration." -tagged( "Bench" ) -see( `function benchmark`, `class SingleBench`, `class MultiBench` ) +tagged( "Bench flow" ) +see( `class SingleBench`, `class MultiBench` ) since( "0.7.0" ) by( "Lis" ) shared interface BenchFlow given Parameter satisfies Anything[] @@ -11,13 +11,15 @@ shared interface BenchFlow "Setups the benchmark test. Called before the test." shared formal void setup(); - "Called before each [[bench]] execution." + "Called before each [[bench]] execution. + Might be called concurrently." shared formal void before(); - "Benchmark function." + "Benchmark function. Is going to be called concurrently." shared formal Anything(*Parameter) bench; - "Called after each [[bench]] execution." + "Called after each [[bench]] execution. + Might be called concurrently." shared formal void after(); "Disposes the benchmark test. Called after the test." @@ -27,6 +29,7 @@ shared interface BenchFlow "Just [[bench]] function makes sense." +tagged( "Bench flow" ) since( "0.7.0" ) by( "Lis" ) class EmptyBenchFlow ( shared actual Anything(*Parameter) bench diff --git a/source/herd/asynctest/benchmark/BenchResult.ceylon b/source/herd/asynctest/benchmark/BenchResult.ceylon new file mode 100644 index 0000000..5e70bce --- /dev/null +++ b/source/herd/asynctest/benchmark/BenchResult.ceylon @@ -0,0 +1,66 @@ + + +"Represents results of a one benchmark. + Contains execution statistic and some additional results." +tagged( "Result" ) +see( `class ParameterResult` ) +since( "0.7.0" ) by( "Lis" ) +shared interface BenchResult satisfies Statistic +{ + "Additional execution results as map of `title` -> `value`." + shared formal Map additional; +} + + +"Returns string representation of additional results" +since( "0.7.0" ) by( "Lis" ) +String stringifyAdditionalResults( BenchResult res ) { + StringBuilder str = StringBuilder(); + for ( title->val in res.additional ) { + str.append( "; " ); + str.append( title ); + str.append( "=" ); + switch ( val ) + case ( is Float ) { + str.append( Float.format( val, 0, 3 ) ); + } + case ( is Integer ) { + str.append( val.string ); + } + } + return str.string; +} + + +"Delegates results to a given data." +since( "0.7.0" ) by( "Lis" ) +class SimpleResults ( + Statistic stat, + shared actual Map additional = emptyMap +) + satisfies BenchResult +{ + + shared actual Float max => stat.max; + + shared actual Float mean => stat.mean; + + shared actual Float min => stat.min; + + shared actual Integer size => stat.size; + + shared actual Float standardDeviation => stat.standardDeviation; + + shared actual Float variance => stat.variance; + + shared actual Float sampleVariance => stat.sampleVariance; + + shared actual Float sampleDeviation => stat.sampleDeviation; + + shared actual Float standardError => stat.standardError; + + shared actual Float sampleError => stat.sampleError; + + shared actual Float relativeSampleError => stat.relativeSampleError; + +} diff --git a/source/herd/asynctest/benchmark/Criteria.ceylon b/source/herd/asynctest/benchmark/Criteria.ceylon index bc1739f..c11e5b8 100644 --- a/source/herd/asynctest/benchmark/Criteria.ceylon +++ b/source/herd/asynctest/benchmark/Criteria.ceylon @@ -27,7 +27,7 @@ shared class NumberOfLoops ( "Continues benchmark iterations while relative sample error doesn't exceed [[maxRelativeError]]. - I.e. [[StatisticSummary.relativeSampleError]] is compared against [[maxRelativeError]]." + I.e. [[Statistic.relativeSampleError]] is compared against [[maxRelativeError]]." tagged( "Criteria" ) see( `class Options` ) throws( `class AssertionError`, "Maximum allowed error is <= 0." ) diff --git a/source/herd/asynctest/benchmark/MultiBench.ceylon b/source/herd/asynctest/benchmark/MultiBench.ceylon index 6a00400..57515cf 100644 --- a/source/herd/asynctest/benchmark/MultiBench.ceylon +++ b/source/herd/asynctest/benchmark/MultiBench.ceylon @@ -16,7 +16,7 @@ shared class MultiBench ( satisfies Bench<[Integer+]> { - shared actual StatisticSummary execute( Options options, [Integer+] parameter ) + shared actual BenchResult execute( Options options, [Integer+] parameter ) => object extends ThreadableRunner( options ) { variable RunnableBench[] benchesMemo = []; shared actual RunnableBench[] benches diff --git a/source/herd/asynctest/benchmark/Options.ceylon b/source/herd/asynctest/benchmark/Options.ceylon index 5de051a..9b95ef5 100644 --- a/source/herd/asynctest/benchmark/Options.ceylon +++ b/source/herd/asynctest/benchmark/Options.ceylon @@ -19,6 +19,10 @@ shared class Options ( "Clock factory (instantiated for each bench and each execution thread ). The clock is used to measure time intervals." shared Clock() clock = WallClock, + "Number of loops to run GC. I.e. GC is run after each given number of loops. + If <= 0 GC is never forced. + Default is 100." + shared Integer loopsToRunGC = 100, "`true` if runs when GC has been started has to be skipped and `flase` otherwise." shared Boolean skipGCRuns = true ) {} diff --git a/source/herd/asynctest/benchmark/ParameterResult.ceylon b/source/herd/asynctest/benchmark/ParameterResult.ceylon index 6ea46ef..a8176b8 100644 --- a/source/herd/asynctest/benchmark/ParameterResult.ceylon +++ b/source/herd/asynctest/benchmark/ParameterResult.ceylon @@ -2,13 +2,13 @@ "Benchmark run result for a given [[parameter]] and a set of benches." tagged( "Result" ) -see( `class Result`, `interface Bench` ) +see( `class Result` ) since( "0.7.0" ) by( "Lis" ) shared final class ParameterResult ( "Parameter the test has been run with." shared Parameter parameter, - "Results of the run for the given bench." Map, StatisticSummary> byBenches + "Results of the run for the given bench." Map, BenchResult> byBenches ) - satisfies Map, StatisticSummary> + satisfies Map, BenchResult> given Parameter satisfies Anything[] { @@ -17,7 +17,7 @@ shared final class ParameterResult ( Float findSlowestMean() { Float minVal = byBenches.fold( infinity ) ( - ( Float res, ->StatisticSummary> item ) + ( Float res, ->BenchResult> item ) => if ( item.item.mean < res ) then item.item.mean else res ); memoizedSlowestMean = minVal; @@ -26,7 +26,7 @@ shared final class ParameterResult ( Float findFastestMean() { Float maxVal = byBenches.fold( -infinity ) ( - ( Float res, ->StatisticSummary> item ) + ( Float res, ->BenchResult> item ) => if ( item.item.mean > res ) then item.item.mean else res ); memoizedFastestMean = maxVal; @@ -35,10 +35,10 @@ shared final class ParameterResult ( "`true` if item mean equals to `slowestMean`." - Boolean filterBySlowest( ->StatisticSummary> item ) => item.item.mean == slowestMean; + Boolean filterBySlowest( ->BenchResult> item ) => item.item.mean == slowestMean; "`true` if item mean equals to `fastestMean`." - Boolean filterByFastest( ->StatisticSummary> item ) => item.item.mean == fastestMean; + Boolean filterByFastest( ->BenchResult> item ) => item.item.mean == fastestMean; "Mean value of the slowest benches." @@ -47,7 +47,7 @@ shared final class ParameterResult ( "A list of benches which showed the smallest performance, i.e. mean number of operations per second is smallest." see( `value slowestMean` ) - shared {->StatisticSummary>*} slowest => byBenches.filter( filterBySlowest ); + shared {->BenchResult>*} slowest => byBenches.filter( filterBySlowest ); "Mean value of the fastest benches." see( `value fastest` ) @@ -55,40 +55,17 @@ shared final class ParameterResult ( "A list of benches which showed the fastest performance, i.e. mean number of operations per second is largest." see( `value fastestMean` ) - shared {->StatisticSummary>*} fastest => byBenches.filter( filterByFastest ); - - "Returns map of [[ComparativeStatistic]] related to the given [[benchOrMean]]." - see( `function relativeToSlowest`, `function relativeToFastest` ) - shared Map, ComparativeStatistic> relativeTo ( - "Bench or summary statistic to calculate comparative statistic to." Bench | Float benchOrMean - ) { - if ( exists stat = if ( is Float benchOrMean ) then benchOrMean else get( benchOrMean )?.mean ) { - return byBenches.mapItems ( - ( Bench key, StatisticSummary item ) => ComparativeStatistic( item, stat ) - ); - } - else { - return emptyMap; - } - } - - "Returns map of [[ComparativeStatistic]] related to the slowest bench." - see( `function relativeTo`, `function relativeToFastest` ) - shared Map, ComparativeStatistic> relativeToSlowest() => relativeTo( slowestMean ); - - "Returns map of [[ComparativeStatistic]] related to the fastest bench." - see( `function relativeToSlowest`, `function relativeTo` ) - shared Map, ComparativeStatistic> relativeToFastest() => relativeTo( fastestMean ); - + shared {->BenchResult>*} fastest => byBenches.filter( filterByFastest ); + shared actual Boolean defines( Object key ) => byBenches.defines( key ); - shared actual StatisticSummary? get( Object key ) => byBenches.get( key ); + shared actual BenchResult? get( Object key ) => byBenches.get( key ); - shared actual StatisticSummary|Default getOrDefault( Object key, Default default ) + shared actual BenchResult|Default getOrDefault( Object key, Default default ) => byBenches.getOrDefault( key, default ); - shared actual Iterator->StatisticSummary> iterator() => byBenches.iterator(); + shared actual Iterator->BenchResult> iterator() => byBenches.iterator(); shared actual Integer size => byBenches.size; diff --git a/source/herd/asynctest/benchmark/RandomDataFlow.ceylon b/source/herd/asynctest/benchmark/RandomDataFlow.ceylon new file mode 100644 index 0000000..bf72986 --- /dev/null +++ b/source/herd/asynctest/benchmark/RandomDataFlow.ceylon @@ -0,0 +1,40 @@ +import java.lang { + ThreadLocal +} +import java.util { + Random +} + + +"Randomly chooses bench function argument from [[sourceData]] and pass selected argument to [[benchFunction]]. + The argument is selected independently for each execution thread. + " +tagged( "Bench flow" ) +since( "0.7.0" ) by( "Lis" ) +shared class RandomDataFlow ( + "Bench function." Anything(Argument) benchFunction, + "Source data for random selection and each execution." Argument[] sourceData +) + satisfies BenchFlow<[]> +{ + + ThreadLocal currentData = ThreadLocal(); + Random rnd = Random(); + + + shared actual default void after() {} + + shared actual default void before() { + currentData.set( sourceData[rnd.nextInt( sourceData.size )] ); + } + + shared actual default Anything() bench { + value data = currentData.get(); + return () => benchFunction( data ); + } + + shared actual default void dispose() {} + + shared actual default void setup() {} + +} diff --git a/source/herd/asynctest/benchmark/RandomFlow.ceylon b/source/herd/asynctest/benchmark/RandomFlow.ceylon new file mode 100644 index 0000000..7f05800 --- /dev/null +++ b/source/herd/asynctest/benchmark/RandomFlow.ceylon @@ -0,0 +1,47 @@ +import java.util { + Random +} + + +"Randomly selects benchmark function from the given list each time before execution. + Each function is paired with corresponding _selection probability_. + Which identifies the selection frequency or distribution. + + For example: + `[function1, 0.3], [function2, 0.7]` leads to `function1` calling of 30% and `function2` calling of 70% + from total number of executions. + + If total sum of probabilities does not equal to 1.0 the probability of any function calling is calculated as + ratio of the given probability to the total sum of probabilities. + + Bench functions are selected independently for each execution thread. +" +throws( `class AssertionError`, "selection probability (i.e. second item in each argument tuple) is not positive." ) +tagged( "Bench flow" ) +see( `class SingleBench`, `class MultiBench` ) +since( "0.7.0" ) by( "Lis" ) +shared class RandomFlow ( + shared actual Integer | Integer() iterations, + "A list of functions to be selected. + Each function is paired with corresponding _selection probability_ which has to be positive." + [Anything(*Parameter), Float]+ benchFunctions +) + extends AbstractSelectiveFlow() + given Parameter satisfies Anything[] +{ + + value actualBenches = reducedProbability( benchFunctions ); + Random rnd = Random(); + + + shared actual Anything(*Parameter) select() { + Float probability = rnd.nextFloat(); + for ( item in actualBenches ) { + if ( probability < item[1] ) { + return item[0]; + } + } + return actualBenches.last[0]; + } + +} diff --git a/source/herd/asynctest/benchmark/Result.ceylon b/source/herd/asynctest/benchmark/Result.ceylon index e1f1a78..5b842bc 100644 --- a/source/herd/asynctest/benchmark/Result.ceylon +++ b/source/herd/asynctest/benchmark/Result.ceylon @@ -17,22 +17,22 @@ shared final class Result ( given Parameter satisfies Anything[] { - class BenchMap( Bench bench ) satisfies Map { + class BenchMap( Bench bench ) satisfies Map { Map> outerParameters => byParameter; shared actual Boolean defines( Object key ) => byParameter.get( bench )?.defines( key ) else false; - shared actual StatisticSummary? get( Object key ) => byParameter.get( bench )?.get( key ); + shared actual BenchResult? get( Object key ) => byParameter.get( bench )?.get( key ); - shared actual StatisticSummary|Default getOrDefault( Object key, Default default ) + shared actual BenchResult|Default getOrDefault( Object key, Default default ) => byParameter.get( key )?.get( key ) else default; - shared actual IteratorStatisticSummary> iterator() - => object satisfies IteratorStatisticSummary> { + shared actual IteratorBenchResult> iterator() + => object satisfies IteratorBenchResult> { IteratorParameterResult> paramIterator = byParameter.iterator(); - shared actual StatisticSummary>|Finished next() { + shared actual BenchResult>|Finished next() { if ( is ParameterResult> nn = paramIterator.next() ) { "Results don't contain specified bench." assert( exists ret = nn.item.get( bench ) ); @@ -69,7 +69,7 @@ shared final class Result ( "Returns results for a specific bench." - shared Map forBench( "Bench the results are asked for." Bench bench ) { + shared Map forBench( "Bench the results are asked for." Bench bench ) { if ( exists ret = benches.get( bench ) ) { return ret; } diff --git a/source/herd/asynctest/benchmark/SelectiveFlow.ceylon b/source/herd/asynctest/benchmark/SelectiveFlow.ceylon new file mode 100644 index 0000000..ae9609f --- /dev/null +++ b/source/herd/asynctest/benchmark/SelectiveFlow.ceylon @@ -0,0 +1,15 @@ + + +"Flow which selects another bench function each given number of benchmark iterations. + [[select]] has to return currently selected bench function. + The bench functions are selected independently for each execution thread." +tagged( "Bench flow" ) +see( `class SingleBench`, `class MultiBench` ) +since( "0.7.0" ) by( "Lis" ) +shared class SelectiveFlow ( + shared actual Anything(*Parameter) select(), + shared actual Integer | Integer() iterations = 0 +) + extends AbstractSelectiveFlow() + given Parameter satisfies Anything[] +{} diff --git a/source/herd/asynctest/benchmark/SequentialFlow.ceylon b/source/herd/asynctest/benchmark/SequentialFlow.ceylon new file mode 100644 index 0000000..8b08a53 --- /dev/null +++ b/source/herd/asynctest/benchmark/SequentialFlow.ceylon @@ -0,0 +1,39 @@ +import java.lang { + ThreadLocal +} +import java.util.\ifunction { + Supplier +} + + +"Sequentially selects bench function from the given list independently for the each running thread." +tagged( "Bench flow" ) +see( `class SingleBench`, `class MultiBench` ) +since( "0.7.0" ) by( "Lis" ) +shared class SequentialFlow ( + shared actual Integer | Integer() iterations, + "A list of bench functions to be sequentially selected." Anything(*Parameter)+ benchFunctions +) + extends AbstractSelectiveFlow() + given Parameter satisfies Anything[] +{ + + ThreadLocal current = ThreadLocal.withInitial ( + object satisfies Supplier { + shared actual Integer get() => 0; + } + ); + + shared actual Anything(*Parameter) select() { + Integer cur = current.get(); + if ( exists ret = benchFunctions[cur] ) { + current.set( cur + 1 ); + return ret; + } + else { + current.set( 1 ); + return benchFunctions.first; + } + } + +} diff --git a/source/herd/asynctest/benchmark/SingleBench.ceylon b/source/herd/asynctest/benchmark/SingleBench.ceylon index 5d84779..4273615 100644 --- a/source/herd/asynctest/benchmark/SingleBench.ceylon +++ b/source/herd/asynctest/benchmark/SingleBench.ceylon @@ -1,6 +1,9 @@ import java.lang { System } +import herd.asynctest.internal { + StatisticAggregator +} "Executes test function in a single thread." @@ -44,7 +47,7 @@ shared final class SingleBench ( * Consume result returned by [[bench]] using [[pushToBlackHole]] * Collect statistic in operations per time units " - shared actual StatisticSummary execute ( + shared actual BenchResult execute ( "Options the bench is executed with." Options options, "Execution parameter the benchmark function takes." Parameter parameter ) { @@ -56,6 +59,7 @@ shared final class SingleBench ( StatisticAggregator calculator = StatisticAggregator(); System.gc(); + variable Integer loops = 0; actualBench.setup(); // bench iterations while ( true ) { @@ -67,8 +71,9 @@ shared final class SingleBench ( // number of GC starts before test run Integer numGCBefore = numberOfGCRuns(); // execute the test function + Anything(*Parameter) benchFunction = actualBench.bench; clock.start(); - Anything ret = actualBench.bench( *parameter ); + Anything ret = benchFunction( *parameter ); Float delta = clock.measure( options.timeUnit ); // number of GC starts after test run Integer numGCAfter = numberOfGCRuns(); @@ -90,6 +95,7 @@ shared final class SingleBench ( if ( exists criterion = warmupCriterion ) { if ( criterion.verify( calculator, options.timeUnit ) ) { // warmup round is completed + loops = 0; warmupCriterion = null; calculator.reset(); System.gc(); @@ -102,10 +108,18 @@ shared final class SingleBench ( } } } + + // force GC + if ( options.loopsToRunGC > 0 ) { + if ( ++ loops == options.loopsToRunGC ) { + loops = 0; + System.gc(); + } + } } actualBench.dispose(); - return calculator.result; + return SimpleResults( calculator.result ); } } diff --git a/source/herd/asynctest/benchmark/ThreadableRunner.ceylon b/source/herd/asynctest/benchmark/ThreadableRunner.ceylon index 010df0b..5fa34f4 100644 --- a/source/herd/asynctest/benchmark/ThreadableRunner.ceylon +++ b/source/herd/asynctest/benchmark/ThreadableRunner.ceylon @@ -9,6 +9,9 @@ import java.util.concurrent { import java.util.concurrent.atomic { AtomicInteger } +import herd.asynctest.internal { + StatisticAggregator +} "Base class for family of benches runners in several threads." @@ -47,9 +50,10 @@ abstract class ThreadableRunner( "Options the bench is executed with." Options o // number of GC starts before test run Integer numGCBefore = numberOfGCRuns(); // execute the test function + Anything() benchFunction = bench.bench; clock.start(); Integer threadsBefore = benchThreads.incrementAndGet(); - Anything ret = bench.bench(); + Anything ret = benchFunction(); Integer threadsAfter = benchThreads.andDecrement; Float delta = clock.measure( options.timeUnit ); // number of GC starts after test run @@ -84,14 +88,16 @@ abstract class ThreadableRunner( "Options the bench is executed with." Options o "Executes the benchmark." - shared StatisticSummary execute() { + shared BenchResult execute() { running = true; benchThreads.set( 0 ); StatisticAggregator calculator = StatisticAggregator(); + StatisticAggregator concurrencyLevel = StatisticAggregator(); loopBarrier = CyclicBarrier( benches.size + 1 ); calculateBarrier = CyclicBarrier( benches.size + 1 ); variable CompletionCriterion? warmupCriterion = options.warmupCriterion; Integer totalBenches = benches.size; + variable Integer loops = 0; // start benches for ( item in benches ) { @@ -110,12 +116,14 @@ abstract class ThreadableRunner( "Options the bench is executed with." Options o meanConcurrencyLevel /= totalBenches; meanOperations /= totalBenches; calculator.sample( meanConcurrencyLevel * meanOperations ); + concurrencyLevel.sample( meanConcurrencyLevel ); // completion verifying if ( exists criterion = warmupCriterion ) { if ( criterion.verify( calculator, options.timeUnit ) ) { // warmup round is completed warmupCriterion = null; + loops = 0; calculator.reset(); System.gc(); } @@ -125,9 +133,21 @@ abstract class ThreadableRunner( "Options the bench is executed with." Options o // measure round is completed running = false; calculateBarrier.await(); - return calculator.result; + return SimpleResults ( + calculator.result, + map( {"concurrency"->concurrencyLevel.mean} ) + ); + } + } + + // force GC + if ( options.loopsToRunGC > 0 ) { + if ( ++ loops == options.loopsToRunGC ) { + loops = 0; + System.gc(); } } + benchThreads.set( 0 ); calculateBarrier.await(); } diff --git a/source/herd/asynctest/benchmark/benchmark.ceylon b/source/herd/asynctest/benchmark/benchmark.ceylon index 9f8b18f..4181cd7 100644 --- a/source/herd/asynctest/benchmark/benchmark.ceylon +++ b/source/herd/asynctest/benchmark/benchmark.ceylon @@ -50,7 +50,7 @@ shared Result benchmark ( ); for ( param in params ) { if ( !res.defines( param ) ) { - HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary> ( + HashMap, BenchResult> benchRes = HashMap, BenchResult> ( linked, Hashtable( benches.size ) ); for ( bench in benches ) { @@ -78,7 +78,7 @@ shared Result benchmark ( assert ( is [Bench<[]>+] benches ); HashMap<[], ParameterResult<[]>> res = HashMap<[], ParameterResult<[]>>(); - HashMap, StatisticSummary> benchRes = HashMap, StatisticSummary> ( + HashMap, BenchResult> benchRes = HashMap, BenchResult> ( linked, Hashtable( benches.size ) ); for ( bench in benches ) { diff --git a/source/herd/asynctest/benchmark/contextWriters.ceylon b/source/herd/asynctest/benchmark/contextWriters.ceylon index e13427c..99e55ad 100644 --- a/source/herd/asynctest/benchmark/contextWriters.ceylon +++ b/source/herd/asynctest/benchmark/contextWriters.ceylon @@ -33,6 +33,7 @@ shared void writeAbsolute( AsyncTestContext context, Result( AsyncTestContext context, Result( AsyncTestContext context, Result< String tuShort = " op/" + results.timeUnit.shortString; for ( param->paramRes in results ) { if ( exists paramStr = stringifyParameter( param ) ) { - for ( bench->stat in paramRes.relativeToSlowest() ) { + for ( bench->stat in paramRes ) { context.succeed ( "``bench.title`` with `" + paramStr + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " - + "``stat.relativeMean``% to slowest; " + + "``( 100 * ( stat.mean / paramRes.slowestMean ) + 0.5 ).integer``% to slowest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + "loops = ``stat.size``" + + stringifyAdditionalResults( stat ) ); } } else { - for ( bench->stat in paramRes.relativeToSlowest() ) { + for ( bench->stat in paramRes ) { context.succeed ( "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " - + "``stat.relativeMean``% to slowest; " + + "``( 100 * ( stat.mean / paramRes.slowestMean ) + 0.5 ).integer``% to slowest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + "loops = ``stat.size``" + + stringifyAdditionalResults( stat ) ); } } @@ -100,25 +104,27 @@ shared void writeRelativeToFastest( AsyncTestContext context, Result< String tuShort = " op/" + results.timeUnit.shortString; for ( param->paramRes in results ) { if ( exists paramStr = stringifyParameter( param ) ) { - for ( bench->stat in paramRes.relativeToFastest() ) { + for ( bench->stat in paramRes ) { context.succeed ( "``bench.title`` with `" + paramStr + "`: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " - + "``stat.relativeMean``% to fastest; " + + "``( 100 * ( stat.mean / paramRes.fastestMean ) + 0.5 ).integer``% to fastest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + "loops = ``stat.size``" + + stringifyAdditionalResults( stat ) ); } } else { - for ( bench->stat in paramRes.relativeToFastest() ) { + for ( bench->stat in paramRes ) { context.succeed ( "``bench.title``: mean=``stringifyNumberOfOperations( stat.mean )````tuShort``; " + "dev=``stringifyNumberOfOperations( stat.sampleDeviation )````tuShort``; " - + "``stat.relativeMean``% to fastest; " + + "``( 100 * ( stat.mean / paramRes.fastestMean ) + 0.5 ).integer``% to fastest; " + "error=``Float.format( stat.relativeSampleError * 100, 0, 2 )``%; " + "loops = ``stat.size``" + + stringifyAdditionalResults( stat ) ); } } diff --git a/source/herd/asynctest/benchmark/package.ceylon b/source/herd/asynctest/benchmark/package.ceylon index ab4eb8d..98c02e6 100644 --- a/source/herd/asynctest/benchmark/package.ceylon +++ b/source/herd/asynctest/benchmark/package.ceylon @@ -17,7 +17,21 @@ Each bench has to satisfy [[Bench]] interface. Any particular bench may implement its own testing semantic and purposes. - For instance, [[SingleBench]] is intended to run test function in a single thread. + [[SingleBench]] is intended to run test function in a single thread. + [[MultiBench]] is intended to run test function in a multithread environment. + + + ### Bench flow + + An object which provides more flexible execution control then just bench function. + Has to satisfy [[BenchFlow]] interface. + Both [[SingleBench]] and [[MultiBench]] may take either bench function of an object satisfied [[BenchFlow]] interface. + + There are some impementations of bench flow: + * [[SelectiveFlow]] which reselects bench function each given number of iterations + * [[SequentialFlow]] which reselects bench function one by one from the given sequence + * [[RandomFlow]] which randomly reselects bench function from the given list each given number of iterations + * [[RandomDataFlow]] which randomly chooses bench function argument from the given list each given number of iterations ### Benchmark execution @@ -106,7 +120,7 @@ writeRelativeToFastest ( context, benchmark ( - Options(LocalIterations(20000).or(LocalError(0.002))), + Options(NumberOfLoops(20000).or(ErrorCriterion(0.002)), NumberOfLoops(100).or(ErrorCriterion(0.002))), [SingleBench(\"plus\", plusBenchmarkFunction), SingleBench(\"minus\", minusBenchmarkFunction)], [1, 1], [2, 3], [25, 34] diff --git a/source/herd/asynctest/benchmark/reducedProbability.ceylon b/source/herd/asynctest/benchmark/reducedProbability.ceylon new file mode 100644 index 0000000..3c69bb0 --- /dev/null +++ b/source/herd/asynctest/benchmark/reducedProbability.ceylon @@ -0,0 +1,31 @@ + + +"Reduces probability (second parameter in each tuple) that: + 1. Total sum is 1.0. + 2. Items are sorted by probability. + 3. Second parameter in each returned tuple is a sum of all previous. + This allows to compare the items with random value + and to select corresponding item according to the given distribution. + " +since( "0.7.0" ) by( "Lis" ) +[[Anything(*Parameter), Float]+] reducedProbability( [[Anything(*Parameter), Float]+] source ) + given Parameter satisfies Anything[] +{ + value sortedBenches = source.sort ( + ([Anything(*Parameter), Float] first, [Anything(*Parameter), Float] second ) => first[1] <=> second[1] + ); + variable Float totalProbability = sortedBenches.fold( 0.0 ) ( + ( Float partial, [Anything(*Parameter), Float] item ) { + "Selection probability has to be positive." + assert( item[1] > 0.0 ); + return partial + item[1]; + } + ); + variable Float count = 0.0; + return sortedBenches.collect ( + ( [Anything(*Parameter), Float] item ) { + count += item[1]; + return [item[0], count / totalProbability ]; + } + ); +} \ No newline at end of file diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index 32db8e6..cc0a6fa 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -376,8 +376,8 @@ by( "Lis" ) native( "jvm" ) module herd.asynctest "0.7.0" { import java.base "8"; - shared import ceylon.test "1.3.1"; - import ceylon.collection "1.3.1"; - shared import ceylon.file "1.3.1"; + shared import ceylon.test "1.3.2"; + import ceylon.collection "1.3.2"; + shared import ceylon.file "1.3.2"; import java.management "8"; } From eed0f72d9c1de193bffb70782a82e3f6b9659993 Mon Sep 17 00:00:00 2001 From: Lis Date: Tue, 21 Mar 2017 09:26:06 +0300 Subject: [PATCH 66/72] 1.3.2 --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 19569de..44a37e2 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ is an extension to SDK `ceylon.test` module with following capabilities: * multi-reporting: several failures or successes can be reported for a one particular test execution (test function), each report is represented as test variant and might be marked with `String` title * reporting test results using charts (or plots) +* benchmarks The module is available on [Ceylon Herd](https://herd.ceylon-lang.org/modules/herd.asynctest). Current version is 0.7.0. @@ -18,17 +19,18 @@ Current version is 0.7.0. #### Ceylon compiler / platform -Compiled with Ceylon 1.3.1 +Compiled with Ceylon 1.3.2 Available on JVM only #### Dependencies -* ceylon.collection/1.3.1 -* ceylon.file/1.3.1 shared -* ceylon.language/1.3.1 -* ceylon.test/1.3.1 shared -* java.base/8 JDK +* ceylon.collection/1.3.2 +* ceylon.file/1.3.2 shared +* ceylon.language/1.3.2 +* ceylon.test/1.3.2 shared +* java.base/8 JDK +* java.management/8 JDK #### Usage and documentation From 0e9eb8cb551b4bde3cbca463ff56ec5284b867ee Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 22 Mar 2017 17:54:44 +0300 Subject: [PATCH 67/72] readme --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 44a37e2..59bd8ef 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,7 @@ See usage details in [API documentation](https://modules.ceylon-lang.org/repo/1/ * Test of [Fibonacci numbers calculation](examples/herd/examples/asynctest/fibonacci). Calculation function is executed on separated thread and returns results using `ceylon.promise`. -* [Time scheduler](examples/herd/examples/asynctest/scheduler) testing. -* [Comparative performance](examples/herd/examples/asynctest/mapperformance) - - comparative performance test of Ceylon / Java HashMap and TreeMap. +* [Time scheduler](examples/herd/examples/asynctest/scheduler) testing. * [Benchmarking](examples/herd/examples/asynctest/benchmark) examples of benchmarks. * [Matchers](examples/herd/examples/asynctest/matchers) - matchers usage. * [Parameterized](examples/herd/examples/asynctest/parameterized) - type- and value- parameterized testing. From a80786614811bc94936eab113dbdf3e5c4d768b9 Mon Sep 17 00:00:00 2001 From: Lis Date: Wed, 22 Mar 2017 18:17:56 +0300 Subject: [PATCH 68/72] throws annotation added --- source/herd/asynctest/parameterization/mixingCombinations.ceylon | 1 + .../herd/asynctest/parameterization/permutingCombinations.ceylon | 1 + .../herd/asynctest/parameterization/zippingCombinations.ceylon | 1 + 3 files changed, 3 insertions(+) diff --git a/source/herd/asynctest/parameterization/mixingCombinations.ceylon b/source/herd/asynctest/parameterization/mixingCombinations.ceylon index 6e9b279..fc17eeb 100644 --- a/source/herd/asynctest/parameterization/mixingCombinations.ceylon +++ b/source/herd/asynctest/parameterization/mixingCombinations.ceylon @@ -57,6 +57,7 @@ Integer firstPermutedKind( ArgumentVariants[] variants, Integer from ) { " tagged( "Combinatorial generators" ) see( `function mixing`, `function permutationSource`, `function zippedSource` ) +throws( `class AssertionError`, "Argument variant is not of 'zipped' or 'permutation' kind" ) since( "0.7.0" ) by( "Lis" ) shared TestVariantEnumerator mixingCombinations ( "Variants of the test function arguments." ArgumentVariants[] arguments diff --git a/source/herd/asynctest/parameterization/permutingCombinations.ceylon b/source/herd/asynctest/parameterization/permutingCombinations.ceylon index 991265a..31ad801 100644 --- a/source/herd/asynctest/parameterization/permutingCombinations.ceylon +++ b/source/herd/asynctest/parameterization/permutingCombinations.ceylon @@ -49,6 +49,7 @@ Anything getArgument( ArgumentVariants? args, Integer? index ) { See example in [[permuting]]." tagged( "Combinatorial generators" ) see( `function permuting`, `function permutationSource` ) +throws( `class AssertionError`, "Argument variant is not of 'permutation' kind" ) since( "0.7.0" ) by( "Lis" ) shared TestVariantEnumerator permutingCombinations ( "Variants of the test function arguments." ArgumentVariants[] arguments diff --git a/source/herd/asynctest/parameterization/zippingCombinations.ceylon b/source/herd/asynctest/parameterization/zippingCombinations.ceylon index 043f393..47b9141 100644 --- a/source/herd/asynctest/parameterization/zippingCombinations.ceylon +++ b/source/herd/asynctest/parameterization/zippingCombinations.ceylon @@ -49,6 +49,7 @@ object finishedIndicator extends FinishedIndicator() {} " tagged( "Combinatorial generators" ) see( `function zipping`, `function zippedSource` ) +throws( `class AssertionError`, "Argument variant is not of 'zipped' kind" ) since( "0.7.0" ) by( "Lis" ) shared TestVariantEnumerator zippingCombinations ( "Variants of the test function arguments." ArgumentVariants[] arguments From ea1e8a89bb5674ccdbb00a247a83905329a14b68 Mon Sep 17 00:00:00 2001 From: Lis Date: Fri, 24 Mar 2017 21:15:36 +0300 Subject: [PATCH 69/72] doc --- source/herd/asynctest/benchmark/benchmark.ceylon | 2 +- source/herd/asynctest/benchmark/delegateWriter.ceylon | 2 +- source/herd/asynctest/benchmark/package.ceylon | 7 +++++-- source/herd/asynctest/module.ceylon | 7 +++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/source/herd/asynctest/benchmark/benchmark.ceylon b/source/herd/asynctest/benchmark/benchmark.ceylon index 4181cd7..0fe7074 100644 --- a/source/herd/asynctest/benchmark/benchmark.ceylon +++ b/source/herd/asynctest/benchmark/benchmark.ceylon @@ -34,7 +34,7 @@ void completeBench( Options options ) { Bench is responsible for the execution details and calculation performance statistic." since( "0.7.0" ) by( "Lis" ) shared Result benchmark ( - "Benchmark options of benches executing." + "Options the benches has to be executed with." Options options, "A list of benches to be executed." [Bench+] benches, diff --git a/source/herd/asynctest/benchmark/delegateWriter.ceylon b/source/herd/asynctest/benchmark/delegateWriter.ceylon index 72ea57f..664b964 100644 --- a/source/herd/asynctest/benchmark/delegateWriter.ceylon +++ b/source/herd/asynctest/benchmark/delegateWriter.ceylon @@ -3,7 +3,7 @@ import herd.asynctest { } -"Delegates benchmark run result writing to the given `writers`." +"Delegates writing of the benchmark run results to the given `writers`." tagged( "Writer" ) see( `function benchmark`, `function writeRelativeToSlowest`, `function writeRelativeToFastest`, `function writeAbsolute` ) diff --git a/source/herd/asynctest/benchmark/package.ceylon b/source/herd/asynctest/benchmark/package.ceylon index 98c02e6..483d4d4 100644 --- a/source/herd/asynctest/benchmark/package.ceylon +++ b/source/herd/asynctest/benchmark/package.ceylon @@ -17,8 +17,9 @@ Each bench has to satisfy [[Bench]] interface. Any particular bench may implement its own testing semantic and purposes. - [[SingleBench]] is intended to run test function in a single thread. - [[MultiBench]] is intended to run test function in a multithread environment. + The package contains two implementations: + * [[SingleBench]] is intended to run test function in a single thread. + * [[MultiBench]] is intended to run test function in a multithread environment. ### Bench flow @@ -33,6 +34,8 @@ * [[RandomFlow]] which randomly reselects bench function from the given list each given number of iterations * [[RandomDataFlow]] which randomly chooses bench function argument from the given list each given number of iterations + Both [[SingleBench]] and [[MultiBench]] benches run test with either benchmark function or [[BenchFlow]]. + ### Benchmark execution diff --git a/source/herd/asynctest/module.ceylon b/source/herd/asynctest/module.ceylon index cc0a6fa..1555351 100644 --- a/source/herd/asynctest/module.ceylon +++ b/source/herd/asynctest/module.ceylon @@ -12,6 +12,7 @@ * multi-reporting: several failures or successes can be reported for a one particular test execution (test function), each report is represented as test variant and might be marked with \`String\` title * reporting test results using charts (or plots) + * benchmarks The extension is based on: @@ -24,7 +25,9 @@ and for modification of the test behaviour. * [[package herd.asynctest.runner]] package which provides a control over a test function execution. * [[package herd.asynctest.match]] package which contains matching API. - * [[package herd.asynctest.chart]] package which is intended to organize reporting with charts. + * [[package herd.asynctest.chart]] package which is intended to organize reporting with charts. + * [[package herd.asynctest.benchmark]] package which contains library for thebenchmark testing. + It is recommended to read documentation on `module ceylon.test` before starting with **asyncTest**. @@ -238,7 +241,7 @@ For example: * A package has three test classes. - * Two of them are annotated with [[concurrent]] and third is not annotated. + * Two of them are annotated with [[concurrent]] and third is not. * Two marked suites are executed via thread pool. Each suite in separated thread if number of available cores admits. But all test functions in the given suite are executed sequentially via a one thread. * After completion the test of the first two suites the third one is executed. From 93abd6591cc8892bdf2ed12fdbb7cf671f6acdb3 Mon Sep 17 00:00:00 2001 From: Lis Date: Sat, 25 Mar 2017 18:47:33 +0300 Subject: [PATCH 70/72] doc --- source/herd/asynctest/chart/package.ceylon | 2 +- source/herd/asynctest/rule/package.ceylon | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/herd/asynctest/chart/package.ceylon b/source/herd/asynctest/chart/package.ceylon index b5aa393..3c10b46 100644 --- a/source/herd/asynctest/chart/package.ceylon +++ b/source/herd/asynctest/chart/package.ceylon @@ -1,5 +1,5 @@ " - [[Chart]] is a set of plots, where each plot is a sequence of 2D points, [[Plot]]. + [[Chart]] is a set of plots, where each plot is a sequence of 2D points. Chart may have title, names of each axis and optional format used by report processor ([[ReportFormat]]). diff --git a/source/herd/asynctest/rule/package.ceylon b/source/herd/asynctest/rule/package.ceylon index e62b61c..f9c6100 100644 --- a/source/herd/asynctest/rule/package.ceylon +++ b/source/herd/asynctest/rule/package.ceylon @@ -1,5 +1,5 @@ " - Test rule provides a way to perform initialization / disposing before / after test execution and to modify test behaviour. + Test rule provides a way to perform test initialization/disposing and a way to modify test behaviour. Each rule is a top-level value or class attribute which satisfies some of the following interfaces: From 4bc5886804837484f1a647fba665e1b8def65b2f Mon Sep 17 00:00:00 2001 From: Lis Date: Sat, 25 Mar 2017 20:02:14 +0300 Subject: [PATCH 71/72] doc --- .../herd/asynctest/benchmark/package.ceylon | 67 ++++++++++--------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/source/herd/asynctest/benchmark/package.ceylon b/source/herd/asynctest/benchmark/package.ceylon index 483d4d4..4b4aeec 100644 --- a/source/herd/asynctest/benchmark/package.ceylon +++ b/source/herd/asynctest/benchmark/package.ceylon @@ -37,11 +37,45 @@ Both [[SingleBench]] and [[MultiBench]] benches run test with either benchmark function or [[BenchFlow]]. + ### Writing benchmark result + + Benchmark run results may be writed to `AsyncTestContext` using a number of writers. See functions tagged as `Writer`. + + ### Benchmark execution [[benchmark]] is intended to run benchmark. The function executes each given bench with each given parameter and returns results of benchmark test as instance of [[Result]]. + Steps to perform benchmark testing: + 1. Define test function or implement [[BenchFlow]] interface. + 2. Choose [[Bench]] which has to run the benchmark. + 3. Invoke [[benchmark]] with the test options, benches and test parameters. + 4. Write benchmark results. + + + ### Example + + Integer plusBenchmarkFunction(Integer x, Integer y) { + return x + y; + } + Integer minusBenchmarkFunction(Integer x, Integer y) { + return x - y; + } + + shared test async void plusMinusBenchmark(AsyncTestContext context) { + writeRelativeToFastest ( + context, + benchmark ( + Options(NumberOfLoops(20000).or(ErrorCriterion(0.002)), NumberOfLoops(100).or(ErrorCriterion(0.002))), + [SingleBench(\"plus\", plusBenchmarkFunction), + SingleBench(\"minus\", minusBenchmarkFunction)], + [1, 1], [2, 3], [25, 34] + ) + ); + context.complete(); + } + ### JIT optimization @@ -86,11 +120,11 @@ There are two constants `x` and `y` in the above examples. JIT may find that result of `+` operation is always the same and replace the last function with - + Integer plusBenchmark() { return 5; } - + So, the plus operation will never be executed. In order to avoid this the parameters might be passed to function as arguments or declared outside the function scope as variables: @@ -103,36 +137,7 @@ > Note: returning value is prefer to direct pushing to black hole, since in this case time consumed by black hole is excluded from total time of the test function execution. - - - ### Writing benchmark result - - Benchmark run results may be writed to `AsyncTestContext` using a number of writers. See functions tagged as `Writer`. - - - ### Example - - Integer plusBenchmarkFunction(Integer x, Integer y) { - return x + y; - } - Integer minusBenchmarkFunction(Integer x, Integer y) { - return x - y; - } - shared test async void plusMinusBenchmark(AsyncTestContext context) { - writeRelativeToFastest ( - context, - benchmark ( - Options(NumberOfLoops(20000).or(ErrorCriterion(0.002)), NumberOfLoops(100).or(ErrorCriterion(0.002))), - [SingleBench(\"plus\", plusBenchmarkFunction), - SingleBench(\"minus\", minusBenchmarkFunction)], - [1, 1], [2, 3], [25, 34] - ) - ); - context.complete(); - } - - " since( "0.7.0" ) by( "Lis" ) shared package herd.asynctest.benchmark; From 0adae15f8847b4d401cd1f906e3d8c83b67c5fd3 Mon Sep 17 00:00:00 2001 From: Lis Date: Mon, 27 Mar 2017 12:31:29 +0300 Subject: [PATCH 72/72] readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 59bd8ef..ff6ca6c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,12 @@ The module is available on [Ceylon Herd](https://herd.ceylon-lang.org/modules/he Current version is 0.7.0. +#### Project structure + +master branch contains the latest release +develop branch contains the currently developed version + + #### Ceylon compiler / platform Compiled with Ceylon 1.3.2