Skip to content

Commit

Permalink
fixes #254: added more detailed test result output on checkpoint fails
Browse files Browse the repository at this point in the history
  • Loading branch information
@S1artie
S1artie committed Feb 4, 2022
1 parent 7fb7b9b commit eafcc60
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 28 deletions.
29 changes: 25 additions & 4 deletions de.gebit.integrity.runner/src/de/gebit/integrity/runner/DefaultTestRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1915,8 +1915,8 @@ protected TestResult executeTest(Test aTest) {
if (tempException == null && aTest.getCheckpoint() != null) {
// In case of checkpoint tests, execution has to be aborted if they fail. This is done by "throwing" an
// abort exception if there is any failed test result.
for (TestComparisonResult tempResult : tempComparisonMap.values()) {
if (tempResult instanceof TestComparisonFailureResult) {
for (Entry<String, TestComparisonResult> tempEntry : tempComparisonMap.entrySet()) {
if (tempEntry.getValue() instanceof TestComparisonFailureResult) {
String tempTestDescription = "<unknown>";
try {
tempTestDescription = testFormatter.testToHumanReadableString(aTest, null);
Expand All @@ -1925,8 +1925,29 @@ protected TestResult executeTest(Test aTest) {
// ignored
}

tempException
= new AbortExecutionException("Checkpoint Test '" + tempTestDescription + "' has failed!");
String tempFailedComparison = "";
try {
tempFailedComparison = "'"
+ valueConverter
.convertValueToString(
(tempEntry.getValue().getExpectedValue() == null ? true
: tempEntry.getValue().getExpectedValue()),
false, null)
+ "' expected"
+ (tempEntry.getKey().equals(ParameterUtil.DEFAULT_PARAMETER_NAME) ? ""
: " for '" + tempEntry.getKey() + "'")
+ ", but got '"
+ valueConverter.convertValueToFormattedString(tempEntry.getValue().getActualValue(),
false,
new ConversionContext().withComparisonResult(tempEntry.getValue().getResult()))
+ "'!";
} catch (Throwable exc) {
tempFailedComparison
= "(unexpected exception during test result formatting: " + exc.getMessage() + ")";
}

tempException = new AbortExecutionException(
"Checkpoint Test '" + tempTestDescription + "' has failed: " + tempFailedComparison);
break;
}
}
Expand Down
26 changes: 13 additions & 13 deletions ...tegrity.tests/results/integrity.basic.exceptions.checkpointTests.checkpointTestNested.xml
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<integrity name="Integrity JUnit Testing" timestamp="14.01.16 10:58" isotimestamp="2016-01-14T10:58:51" duration="4.987" abortMessage="Checkpoint Test 'Echo the string 'foo'' has failed!">
<integrity name="Integrity JUnit Testing" timestamp="04.02.22 12:58" isotimestamp="2022-02-04T12:58:39" duration="22.628" abortMessage="Checkpoint Test 'Echo the string 'foo'' has failed: 'bar' expected, but got 'foo'!">
<variables />
<suite id="0" name="integrity.basic.exceptions.checkpointTests.checkpointTestNested" timestamp="14.01.16 10:58:51.0620">
<suite id="0" name="integrity.basic.exceptions.checkpointTests.checkpointTestNested" timestamp="04.02.22 12:58:39.0142">
<setup />
<variables />
<statements>
<suite id="1" line="17" name="integrity.basic.exceptions.checkpointTests.innerCheckpointTestNested" timestamp="14.01.16 10:58:51.0620">
<suite id="1" line="17" name="integrity.basic.exceptions.checkpointTests.innerCheckpointTestNested" timestamp="04.02.22 12:58:39.0145">
<setup />
<variables />
<statements>
<call id="2" line="22" name="noOp" description="Do absolutely nothing." fixture="de.gebit.integrity.tests.fixtures.basic.NoOpFixture#noOp" timestamp="14.01.16 10:58:51.0620">
<call id="2" line="22" name="noOp" description="Do absolutely nothing." fixture="de.gebit.integrity.tests.fixtures.basic.NoOpFixture#noOp" timestamp="04.02.22 12:58:39.0146">
<parameters />
<result duration="0.023" type="success" />
<result duration="0.379" type="success" />
</call>
<test id="3" line="25" name="echoStringTest" description="Echo the string 'foo'" fixture="de.gebit.integrity.tests.fixtures.basic.NoOpFixture#echoString" timestamp="14.01.16 10:58:51.0620">
<results duration="0.160" successCount="1" failureCount="0" exceptionCount="0">
<result duration="0.160" description="Echo the string 'foo'" type="success">
<test id="3" line="25" name="echoStringTest" description="Echo the string 'foo'" fixture="de.gebit.integrity.tests.fixtures.basic.NoOpFixture#echoString" timestamp="04.02.22 12:58:39.0152">
<results duration="2.295" successCount="1" failureCount="0" exceptionCount="0">
<result duration="2.295" description="Echo the string 'foo'" type="success">
<parameters>
<parameter name="string" value="foo" />
</parameters>
Expand All @@ -25,9 +25,9 @@
</result>
</results>
</test>
<test id="4" line="28" name="echoStringTest" description="Echo the string 'foo'" fixture="de.gebit.integrity.tests.fixtures.basic.NoOpFixture#echoString" timestamp="14.01.16 10:58:51.0620">
<results duration="0.098" successCount="0" failureCount="0" exceptionCount="1">
<result duration="0.098" description="Echo the string 'foo'" type="exception" exceptionMessage="Checkpoint Test 'Echo the string 'foo'' has failed!" exceptionTrace="de.gebit.integrity.exceptions.AbortExecutionException: Checkpoint Test 'Echo the string 'foo'' has failed!&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.executeTest(DefaultTestRunner.java:1400)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.executeSuite(DefaultTestRunner.java:1015)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.callSuiteSingle(DefaultTestRunner.java:805)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.callSuite(DefaultTestRunner.java:682)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.executeSuite(DefaultTestRunner.java:1011)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.callSuiteSingle(DefaultTestRunner.java:805)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.runInternal(DefaultTestRunner.java:635)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.run(DefaultTestRunner.java:513)&#xD;&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.run(DefaultTestRunner.java:448)&#xD;&#xA;&#x9;at de.gebit.integrity.tests.junit.IntegrityJUnitTest.executeIntegritySuite(IntegrityJUnitTest.java:99)&#xD;&#xA;&#x9;at de.gebit.integrity.tests.junit.basic.exceptions.CheckpointTests.checkpointTestNested(CheckpointTests.java:51)&#xD;&#xA;&#x9;at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)&#xD;&#xA;&#x9;at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)&#xD;&#xA;&#x9;at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)&#xD;&#xA;&#x9;at java.lang.reflect.Method.invoke(Method.java:606)&#xD;&#xA;&#x9;at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)&#xD;&#xA;&#x9;at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)&#xD;&#xA;&#x9;at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)&#xD;&#xA;&#x9;at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)&#xD;&#xA;&#x9;at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)&#xD;&#xA;&#x9;at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)&#xD;&#xA;&#x9;at org.junit.runners.ParentRunner.run(ParentRunner.java:363)&#xD;&#xA;&#x9;at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)&#xD;&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)&#xD;&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)&#xD;&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675)&#xD;&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)&#xD;&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)&#xD;&#xA;">
<test id="4" line="28" name="echoStringTest" description="Echo the string 'foo'" fixture="de.gebit.integrity.tests.fixtures.basic.NoOpFixture#echoString" timestamp="04.02.22 12:58:39.0159">
<results duration="0.271" successCount="0" failureCount="0" exceptionCount="1">
<result duration="0.271" description="Echo the string 'foo'" type="exception" exceptionMessage="Checkpoint Test 'Echo the string 'foo'' has failed: 'bar' expected, but got 'foo'!" exceptionTrace="de.gebit.integrity.exceptions.AbortExecutionException: Checkpoint Test 'Echo the string 'foo'' has failed: 'bar' expected, but got 'foo'!&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.executeTest(DefaultTestRunner.java:1949)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.executeSuite(DefaultTestRunner.java:1460)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.callSuiteSingle(DefaultTestRunner.java:1154)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.callSuite(DefaultTestRunner.java:1013)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.executeSuite(DefaultTestRunner.java:1456)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.callSuiteSingle(DefaultTestRunner.java:1154)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner$6.run(DefaultTestRunner.java:957)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner$6.run(DefaultTestRunner.java:1)&#xA;&#x9;at de.gebit.integrity.runner.logging.TestRunnerPerformanceLogger.executeAndLog(TestRunnerPerformanceLogger.java:101)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.runInternal(DefaultTestRunner.java:952)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.run(DefaultTestRunner.java:819)&#xA;&#x9;at de.gebit.integrity.runner.DefaultTestRunner.run(DefaultTestRunner.java:709)&#xA;&#x9;at de.gebit.integrity.tests.junit.IntegrityJUnitTest.executeIntegritySuite(IntegrityJUnitTest.java:176)&#xA;&#x9;at de.gebit.integrity.tests.junit.IntegrityJUnitTest.executeIntegritySuite(IntegrityJUnitTest.java:127)&#xA;&#x9;at de.gebit.integrity.tests.junit.basic.exceptions.CheckpointTests.checkpointTestNested(CheckpointTests.java:51)&#xA;&#x9;at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)&#xA;&#x9;at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)&#xA;&#x9;at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)&#xA;&#x9;at java.lang.reflect.Method.invoke(Method.java:498)&#xA;&#x9;at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)&#xA;&#x9;at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)&#xA;&#x9;at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)&#xA;&#x9;at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)&#xA;&#x9;at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)&#xA;&#x9;at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)&#xA;&#x9;at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)&#xA;&#x9;at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)&#xA;&#x9;at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)&#xA;&#x9;at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)&#xA;&#x9;at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)&#xA;&#x9;at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)&#xA;&#x9;at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)&#xA;&#x9;at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)&#xA;&#x9;at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)&#xA;&#x9;at org.junit.runners.ParentRunner.run(ParentRunner.java:413)&#xA;&#x9;at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:89)&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:41)&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:542)&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:770)&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:464)&#xA;&#x9;at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:210)&#xA;">
<parameters>
<parameter name="string" value="foo" />
</parameters>
Expand All @@ -37,12 +37,12 @@
</statements>
<returns />
<teardown />
<result duration="4.227" successCount="1" failureCount="0" exceptionCount="1" testExceptionCount="1" callExceptionCount="0" />
<result duration="18.787" successCount="1" failureCount="0" exceptionCount="1" testExceptionCount="1" callExceptionCount="0" />
</suite>
</statements>
<returns />
<teardown />
<result duration="4.560" successCount="1" failureCount="0" exceptionCount="1" testExceptionCount="1" callExceptionCount="0" />
<result duration="19.901" successCount="1" failureCount="0" exceptionCount="1" testExceptionCount="1" callExceptionCount="0" />
</suite>
</integrity>

Loading

0 comments on commit eafcc60

Please sign in to comment.