From 6afbc302d99e50650daef71c47d4e491ae7310f7 Mon Sep 17 00:00:00 2001
From: Hendrik Cannoodt <hendrik.cannoodt@gmail.com>
Date: Fri, 13 Oct 2023 11:25:51 +0200
Subject: [PATCH] Feature/local dependency test2 (#565)

* Test local repositories

Test with both absolute and relative paths.

* Refactor TestHelper class

Extend ExceptionOutput with exitCode and use it in all functions

Remove testMain and replace it with testMainWithStdErr functionality and rename it testMain as the sole function
Combine functionality of testMainException and testMainException2

* Fix modified testbench checking the wrong output for text

* Fix cherry picking

* Add changelog entry

* Fix merge
---
 CHANGELOG.md                                  |   6 +
 src/test/scala/io/viash/TestHelper.scala      |  79 ++----
 .../MainBuildAuxiliaryDockerChown.scala       |   4 +-
 ...MainBuildAuxiliaryDockerRequirements.scala |  18 +-
 ...nBuildAuxiliaryDockerResourceCopying.scala |   4 +-
 .../MainTestAuxiliaryDockerResourceCopy.scala |  24 +-
 .../io/viash/e2e/build/DockerMoreSuite.scala  |  22 +-
 .../io/viash/e2e/build/NativeSuite.scala      |  13 +-
 .../e2e/config_view/MainConfigViewSuite.scala |  12 +-
 .../io/viash/e2e/export/MainExportSuite.scala |  78 +++---
 .../io/viash/e2e/help/MainHelpSuite.scala     |  38 +--
 .../e2e/ns_build/MainNSBuildNativeSuite.scala |   6 +-
 .../e2e/ns_exec/MainNSExecNativeSuite.scala   |  13 +-
 .../e2e/ns_list/MainNSListNativeSuite.scala   | 146 ++++++------
 .../e2e/ns_test/MainNSTestNativeSuite.scala   |  26 +-
 .../run/RunComputationalRequirements.scala    |  54 ++---
 .../viash/e2e/test/MainTestDockerSuite.scala  |  84 +++----
 .../viash/e2e/test/MainTestNativeSuite.scala  | 224 +++++++++---------
 .../test/TestComputationalRequirements.scala  |  54 ++---
 .../dependencies/Dependency.scala             | 122 +++++++++-
 .../runners/nextflow/NextflowScriptTest.scala |   4 +-
 .../runners/nextflow/Vdsl3ModuleTest.scala    |   8 +-
 .../nextflow/Vdsl3StandaloneTest.scala        |   2 +-
 .../runners/nextflow/WorkflowHelperTest.scala |   2 +-
 24 files changed, 565 insertions(+), 478 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 983bad0af..2e89bd247 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,12 @@ TODO add summary
 * `export json_schema`: Add a `--strict` option to output a subset of the schema representing the internal structure of the Viash config (PR #564).
 
 * `config view` and `ns list`: Do not output internal functionality fields (#564). Additionally, add a validation that no internal fields are present when reading a Viash config file.
+
+## MINOR CHANGES
+
+* `testbenches`: Add testbenches for local dependencies (PR #565).
+
+* `testbenches`: Refactor testbenches helper functions to uniformize them (PR #565).
   
 # Viash 0.8.0-RC5 (2023-10-11): Fix run workflow
 
diff --git a/src/test/scala/io/viash/TestHelper.scala b/src/test/scala/io/viash/TestHelper.scala
index b92d11cd5..4224a6669 100644
--- a/src/test/scala/io/viash/TestHelper.scala
+++ b/src/test/scala/io/viash/TestHelper.scala
@@ -10,94 +10,65 @@ import scala.reflect.ClassTag
 
 object TestHelper {
 
-  case class ExceptionOutput(
-    exceptionText: String,
-    output: String,
-    error: String,
+  case class TestMainOutput(
+    stdout: String,
+    stderr: String,
+    exitCode: Option[Int],
+    exceptionText: Option[String],
   )
 
   /**
-   * Method to capture the console stdout generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout being printed to the console, we print it after the Main.main() is finished.
+   * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console
    * @param args all the arguments typically passed to Main.main()
-   * @return a string of all the output
+   * @return TestMainOutput containing the console output text and exit code
    */
-  def testMain(args: String*) : String = {
-    val os = new ByteArrayOutputStream()
-    Console.withErr(os) {
-      Console.withOut(os) {
-        Main.mainCLI(args.toArray)
-      }
-    }
-
-    val stdout = os.toString
-    // Console.print(stdout)
-    stdout
-  }
+  def testMain(args: String*): TestMainOutput = testMain(None, args: _*)
 
   /**
    * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout and stderr being printed to the console, we print it after the Main.main() is finished.
+   * @param workingDir the working directory to run the command in
    * @param args all the arguments typically passed to Main.main()
-   * @return a tuple of stdout and stderr strings of all the output
+   * @return TestMainOutput containing the console output text and exit code
    */
-  def testMainWithStdErr(args: String*) : (String, String, Int) = {
+  def testMain(workingDir: Option[Path], args: String*): TestMainOutput = {
     val outStream = new ByteArrayOutputStream()
     val errStream = new ByteArrayOutputStream()
     val exitCode = Console.withOut(outStream) {
       Console.withErr(errStream) {
-        Main.mainCLI(args.toArray)
+        Main.mainCLI(args.toArray, workingDir)
       }
     }
 
-    val stdout = outStream.toString
-    val stderr = errStream.toString
-    // Console.print(stdout)
-    (stdout, stderr, exitCode)
+    TestMainOutput(outStream.toString, errStream.toString, Some(exitCode), None)
   }
 
   /**
-   * Method to capture the console stdout generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout being printed to the console, we print it after the Main.main() is finished.
-   * Additionally it handles a thrown RuntimeException using assertThrows
+   * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console.
+   * Additionally it handles a thrown Exception/Throwable and returns the exception text.
    * @param args all the arguments typically passed to Main.main()
-   * @return a string of all the output
+   * @return TestMainOutput containing the exception text and the console output text
    */
-  def testMainException[T <: AnyRef: ClassTag](args: String*) : String = {
-    val os = new ByteArrayOutputStream()
-    assertThrows[T] {
-      Console.withOut(os) {
-        Main.mainCLI(args.toArray)
-      }
-    }
-
-    val stdout = os.toString
-    // Console.print(stdout)
-    stdout
-  }
+  def testMainException[T <: Throwable](args: String*)(implicit classTag: ClassTag[T]) : TestMainOutput = testMainException(None, args: _*)
 
   /**
-   * Method to capture the console stdout generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout being printed to the console, we print it after the Main.main() is finished.
-   * Additionally it handles a thrown RuntimeException using assertThrows
+   * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console.
+   * Additionally it handles a thrown Exception/Throwable and returns the exception text.
+   * @param workingDir the working directory to run the command in
    * @param args all the arguments typically passed to Main.main()
-   * @return ExceptionOutput containing the exception text and the console output text
+   * @return TestMainOutput containing the exception text and the console output text
    */
-  def testMainException2[T <: Exception](args: String*) : ExceptionOutput = {
+  def testMainException[T <: Throwable](workingDir: Option[Path], args: String*)(implicit classTag: ClassTag[T]) : TestMainOutput = {
     val outStream = new ByteArrayOutputStream()
     val errStream = new ByteArrayOutputStream()
-    val caught = intercept[Exception] {
+    val caught = intercept[T] {
       Console.withOut(outStream) {
         Console.withErr(errStream) {
-          Main.mainCLI(args.toArray)
+          Main.mainCLI(args.toArray, workingDir)
         }
       }
     }
 
-    val stdout = outStream.toString
-    val stderr = errStream.toString
-    // Console.print(stdout)
-    ExceptionOutput(caught.getMessage, stdout, stderr)
+    TestMainOutput(outStream.toString, errStream.toString, None, Some(caught.getMessage))
   }
 
 
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala
index 426206efc..73749d4f8 100644
--- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala
+++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala
@@ -116,13 +116,13 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll w
   test("Test with platform and chown is set to false", DockerTest) {
     val newConfigFile = configDeriver.derive(""".platforms := [{type: "docker", chown: false }]""", "docker_chown_false")
     // functionality not provided in runner, should throw exception
-    val output = TestHelper.testMainException2[ConfigParserException](
+    val output = TestHelper.testMainException[ConfigParserException](
       "build",
       "--engine", "docker_chown",
       "-o", tempFolStr,
       newConfigFile
     )
-    assert(output.error.contains("Error: ..chown was removed: Compability not provided with the Runners functionality."))
+    assert(output.stderr.contains("Error: ..chown was removed: Compability not provided with the Runners functionality."))
   }
 
   override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
index c7d16867d..3242a4d4d 100644
--- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
+++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
@@ -568,29 +568,29 @@ class MainBuildAuxiliaryDockerRequirementsApkTest extends AbstractMainBuildAuxil
   test("test_setup; check the fortune package is added for the test option", DockerTest) { f =>
     val newConfigFilePath = deriveEngineConfig(None, Some("""[{ "type": "apk", "packages": ["fortune"] }]"""), "apk_test_fortune_test")
 
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       newConfigFilePath
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 1 out of 1 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 1 out of 1 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
   }
 
   test("test_setup; check the fortune package is not added for the test option when not specified", DockerTest) { f =>
     val newConfigFilePath = deriveEngineConfig(None, None, "apk_base_test")
 
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "-k", "false",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText == "Only 0 out of 1 test scripts succeeded!")
+    assert(testOutput.exceptionText.get == "Only 0 out of 1 test scripts succeeded!")
 
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(testOutput.output.contains("ERROR! Only 0 out of 1 test scripts succeeded!"))
-    assert(testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Only 0 out of 1 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
   }
 }
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala
index ff17f663e..98690930c 100644
--- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala
+++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala
@@ -77,14 +77,14 @@ class MainBuildAuxiliaryDockerResourceCopying extends AnyFunSuite with BeforeAnd
   test("Check resources with unsupported format") {
     val configResourcesUnsupportedProtocolFile = configDeriver.derive(""".functionality.resources := [{type: "bash_script", path: "./check_bash_version.sh"}, {path: "ftp://ftp.ubuntu.com/releases/robots.txt"}]""", "config_resource_unsupported_protocol").toString
     // generate viash script
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "build",
       "--engine", "docker",
       "-o", tempFolStr,
       configResourcesUnsupportedProtocolFile
     )
 
-    assert(testOutput.exceptionText == "Unsupported scheme: ftp")
+    assert(testOutput.exceptionText.get == "Unsupported scheme: ftp")
   }
 
   override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
index d968c251a..e9e6228ef 100644
--- a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
+++ b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
@@ -29,7 +29,7 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
     Files.copy(tmpFolderResourceSourceFile, tmpFolderResourceDestinationFile, StandardCopyOption.REPLACE_EXISTING)
 
     // generate viash script
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "-k", "true",
@@ -37,13 +37,13 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
     )
 
     // basic checks to see if standard test/build was correct
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("WARNING! No tests found!"))
-    assert(!testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("WARNING! No tests found!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
     val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r
 
-    val tempPath = testText.replaceAll("\n", "") match {
+    val tempPath = testOutput.stdout.replaceAll("\n", "") match {
       case FolderRegex(path) => path
       case _ => ""
     }
@@ -77,27 +77,27 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
     }
 
     Directory(tmpFolderResourceDestinationFolder).deleteRecursively()
-    checkTempDirAndRemove(testText, true, "viash_test_auxiliary_resources")
+    checkTempDirAndRemove(testOutput.stdout, true, "viash_test_auxiliary_resources")
   }
 
   test("Check resources with unsupported format", DockerTest) {
     val configResourcesUnsupportedProtocolFile = configDeriver.derive(""".functionality.resources := [{type: "bash_script", path: "./check_bash_version.sh"}, {path: "ftp://ftp.ubuntu.com/releases/robots.txt"}]""", "config_resource_unsupported_protocol").toString
     // generate viash script
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "docker",
       "-k", "true",
       configResourcesUnsupportedProtocolFile
     )
 
-    assert(testOutput.exceptionText == "Unsupported scheme: ftp")
+    assert(testOutput.exceptionText.get == "Unsupported scheme: ftp")
 
     // basic checks to see if standard test/build was correct
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(!testOutput.output.contains("WARNING! No tests found!"))
-    assert(!testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(!testOutput.stdout.contains("WARNING! No tests found!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testOutput.output, true, "viash_test_auxiliary_resources")
+    checkTempDirAndRemove(testOutput.stdout, true, "viash_test_auxiliary_resources")
   }
 
   /**
diff --git a/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala b/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala
index ff9e505cd..de40089fa 100644
--- a/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala
+++ b/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala
@@ -36,7 +36,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "commands_default"
     )
     
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -45,7 +45,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), stdout)
+    assert(testOutput.stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), testOutput.stdout)
   }
 
   test("Verify adding extra commands to verify", DockerTest) {
@@ -54,7 +54,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "commands_extra"
     )
     
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -63,7 +63,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), stdout)
+    assert(testOutput.stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), testOutput.stdout)
   }
 
   test("Verify base adding an extra required command that doesn't exist", DockerTest) {
@@ -72,7 +72,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "non_existing_command"
     )
     
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -81,14 +81,14 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stdout.contains("[notice] Building container 'testbash:0.1' with Dockerfile"))
-    assert(stdout.contains("[error] Docker container 'testbash:0.1' does not contain command 'non_existing_command'."))
+    assert(testOutput.stdout.contains("[notice] Building container 'testbash:0.1' with Dockerfile"))
+    assert(testOutput.stdout.contains("[error] Docker container 'testbash:0.1' does not contain command 'non_existing_command'."))
   }
 
   test("Check deprecated warning", DockerTest) {
     val newConfigFilePath = configDeriver.derive(""".functionality.status := "deprecated"""", "deprecated")
     
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -97,8 +97,8 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stderr.contains("The status of the component 'testbash' is set to deprecated."))
-    assert(exitCode == 0)
+    assert(testOutput.stderr.contains("The status of the component 'testbash' is set to deprecated."))
+    assert(testOutput.exitCode == Some(0))
   }
 
   test("Check component works when multiple_sep is set to ;", DockerTest) {
@@ -107,7 +107,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "multiple_sep"
     )
     
-    val _ = TestHelper.testMainWithStdErr(
+    val _ = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
diff --git a/src/test/scala/io/viash/e2e/build/NativeSuite.scala b/src/test/scala/io/viash/e2e/build/NativeSuite.scala
index c0521919b..bc0e34d9a 100644
--- a/src/test/scala/io/viash/e2e/build/NativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/build/NativeSuite.scala
@@ -10,6 +10,7 @@ import io.viash.config.Config
 
 import scala.io.Source
 import io.viash.helpers.{IO, Exec, Logger}
+import io.viash.exceptions.ConfigParserException
 
 class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
   Logger.UseColorOverride.value = Some(false)
@@ -222,11 +223,11 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
     )
 
     val testRegex = "Warning: ..platforms is deprecated: Use 'engines' and 'runners' instead.".r
-    assert(testRegex.findFirstIn(testOutput).isDefined, testOutput)
+    assert(testRegex.findFirstIn(testOutput.stderr).isDefined, testOutput)
   }
 
   test("Test whether defining strings as arguments in argument groups throws a removed error") {
-    val testOutput = TestHelper.testMainException2[Exception](
+    val testOutput = TestHelper.testMainException[Exception](
       "build",
       "-o", tempFolStr,
       configDeprecatedArgumentGroups
@@ -241,19 +242,19 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
     assert(out.exitValue == 0)
 
     val testRegex = "Error: specifying strings in the .argument field of argument group 'First group' was removed.".r
-    assert(testRegex.findFirstIn(testOutput.error).isDefined, testOutput.error)
+    assert(testRegex.findFirstIn(testOutput.stderr).isDefined, testOutput.stderr)
   }
 
   test("Test whether setting an internalFunctionality field throws an error") {
     val newConfigFilePath = configDeriver.derive(""".functionality.argument_groups[.name == "First group"].arguments[.name == "input"].dest := "foo"""", "set_internal_functionality")
 
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "build",
       "-o", tempFolStr,
       newConfigFilePath
     )
 
-    assert(testOutput.error.contains("Error: .functionality.argument_groups.arguments.dest is internal functionality."))
+    assert(testOutput.stderr.contains("Error: .functionality.argument_groups.arguments.dest is internal functionality."))
   }
 
   test("Test config without a main script") {
@@ -264,7 +265,7 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       "-c", ".functionality.resources := []"
     )
 
-    assert(testOutput.contains("Warning: no resources specified!"))
+    assert(testOutput.stderr.contains("Warning: no resources specified!"))
   }
 
   override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala b/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala
index ba31035c0..65a1d4a83 100644
--- a/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala
+++ b/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala
@@ -13,23 +13,23 @@ class MainConfigViewSuite extends AnyFunSuite{
 
 
   test("viash config view local") {
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "config", "view",
       configFile
     )
 
-    assert(stdout.startsWith("functionality:"))
-    assert(stdout.contains("testbash"))
+    assert(testOutput.stdout.startsWith("functionality:"))
+    assert(testOutput.stdout.contains("testbash"))
   }
 
   test("viash config view remote") {
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "config", "view",
       "https://raw.githubusercontent.com/viash-io/viash/develop/src/test/resources/testbash/config.vsh.yaml"
     )
 
-    assert(stdout.startsWith("functionality:"))
-    assert(stdout.contains("testbash"))
+    assert(testOutput.stdout.startsWith("functionality:"))
+    assert(testOutput.stdout.contains("testbash"))
   }
 
 }
diff --git a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala
index 04f073df8..fb1ffedca 100644
--- a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala
+++ b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala
@@ -23,15 +23,15 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   // These are all *very* basic tests. Practicly no validation whatsoever to check whether the output is correct or not.
 
   test("viash export resource") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "resource", "runners/nextflow/WorkflowHelper.nf"
     )
 
-    assert(stdout.contains("def readConfig("))
+    assert(testOutput.stdout.contains("def readConfig("))
   }
 
   test("viash export resource to file") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "resource", "runners/nextflow/WorkflowHelper.nf",
       "--output", tempFile.toString
     )
@@ -41,38 +41,38 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export resource legacy") {
-    val (stdout, stderr, code) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "export", "resource", "platforms/nextflow/WorkflowHelper.nf"
     )
 
-    assert(stderr.contains("WARNING: The 'platforms/' prefix is deprecated. Please use 'runners/' instead."))
+    assert(testOutput.stderr.contains("WARNING: The 'platforms/' prefix is deprecated. Please use 'runners/' instead."))
 
-    assert(stdout.contains("def readConfig("))
+    assert(testOutput.stdout.contains("def readConfig("))
   }
 
   test("viash export resource to file legacy") {
-    val (stdout, stderr, code) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "export", "resource", "platforms/nextflow/WorkflowHelper.nf",
       "--output", tempFile.toString
     )
 
-    assert(stderr.contains("WARNING: The 'platforms/' prefix is deprecated. Please use 'runners/' instead."))
+    assert(testOutput.stderr.contains("WARNING: The 'platforms/' prefix is deprecated. Please use 'runners/' instead."))
 
     val lines = helpers.IO.read(tempFile.toUri())
     assert(lines.contains("def readConfig("))
   }
   
   test("viash export cli_schema") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "cli_schema"
     )
 
-    assert(stdout.startsWith("""- name: "run""""))
-    assert(stdout.contains("viash config inject"))
+    assert(testOutput.stdout.startsWith("""- name: "run""""))
+    assert(testOutput.stdout.contains("viash config inject"))
   }
   
   test("viash export cli_schema to file") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "cli_schema",
       "--output", tempFile.toString
     )
@@ -83,16 +83,16 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export cli_autocomplete without format") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "cli_autocomplete"
     )
 
-    assert(stdout.startsWith("""# bash completion for viash"""))
-    assert(stdout.contains("COMPREPLY=($(compgen -W 'run build test ns config' -- \"$cur\"))"))
+    assert(testOutput.stdout.startsWith("""# bash completion for viash"""))
+    assert(testOutput.stdout.contains("COMPREPLY=($(compgen -W 'run build test ns config' -- \"$cur\"))"))
   }
 
   test("viash export cli_autocomplete without format to file") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "cli_autocomplete",
       "--output", tempFile.toString
     )
@@ -103,17 +103,17 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export cli_autocomplete Bash") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "cli_autocomplete",
       "--format", "bash"
     )
 
-    assert(stdout.startsWith("""# bash completion for viash"""))
-    assert(stdout.contains("COMPREPLY=($(compgen -W 'run build test ns config' -- \"$cur\"))"))
+    assert(testOutput.stdout.startsWith("""# bash completion for viash"""))
+    assert(testOutput.stdout.contains("COMPREPLY=($(compgen -W 'run build test ns config' -- \"$cur\"))"))
   }
 
   test("viash export cli_autocomplete Bash to file") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "cli_autocomplete",
       "--format", "bash",
       "--output", tempFile.toString
@@ -125,17 +125,17 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export cli_autocomplete Zsh") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "cli_autocomplete",
       "--format", "zsh"
     )
 
-    assert(stdout.startsWith("""#compdef viash"""))
-    assert(stdout.contains("_viash_export_commands"))
+    assert(testOutput.stdout.startsWith("""#compdef viash"""))
+    assert(testOutput.stdout.contains("_viash_export_commands"))
   }
 
   test("viash export cli_autocomplete Zsh to file") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "cli_autocomplete",
       "--format", "zsh",
       "--output", tempFile.toString
@@ -147,16 +147,16 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export config_schema") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "config_schema"
     )
 
-    assert(stdout.startsWith("""- - name: "__this__""""))
-    assert(stdout.contains("""type: "OneOrMore[String]""""))
+    assert(testOutput.stdout.startsWith("""- - name: "__this__""""))
+    assert(testOutput.stdout.contains("""type: "OneOrMore[String]""""))
   }
 
   test("viash export config_schema to file") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "config_schema",
       "--output", tempFile.toString
     )
@@ -167,25 +167,25 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export json_schema") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "json_schema"
     )
 
-    assert(stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
-    assert(stdout.contains("""- $ref: "#/definitions/Config""""))
+    assert(testOutput.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+    assert(testOutput.stdout.contains("""- $ref: "#/definitions/Config""""))
   }
 
   test("viash export json_schema, explicit yaml format") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "json_schema", "--format", "yaml"
     )
 
-    assert(stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
-    assert(stdout.contains("""- $ref: "#/definitions/Config""""))
+    assert(testOutput.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+    assert(testOutput.stdout.contains("""- $ref: "#/definitions/Config""""))
   }
 
   test("viash export json_schema to file, explicit yaml format") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "json_schema", "--format", "yaml",
       "--output", tempFile.toString
     )
@@ -196,20 +196,20 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
   }
 
   test("viash export json_schema, json format") {
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "export", "json_schema", "--format", "json"
     )
 
-    assert(stdout.startsWith(
+    assert(testOutput.stdout.startsWith(
         """{
           |  "$schema" : "https://json-schema.org/draft-07/schema#",
           |  "definitions" : {
           |""".stripMargin))
-    assert(stdout.contains(""""$ref" : "#/definitions/Config""""))
+    assert(testOutput.stdout.contains(""""$ref" : "#/definitions/Config""""))
   }
 
   test("viash export json_schema to file, json format") {
-    val stdout = TestHelper.testMain(
+    TestHelper.testMain(
       "export", "json_schema", "--format", "json",
       "--output", tempFile.toString
     )
diff --git a/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala b/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala
index a02c36f0a..d68012429 100644
--- a/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala
+++ b/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala
@@ -11,69 +11,69 @@ class MainHelpSuite extends AnyFunSuite{
   private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath
 
   test("viash config view default functionality without help") {
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "config", "view",
       configFile
     )
 
-    assert(stdout.startsWith("functionality:"))
-    assert(stdout.contains("testbash"))
+    assert(testOutput.stdout.startsWith("functionality:"))
+    assert(testOutput.stdout.contains("testbash"))
   }
 
   test("viash config view default functionality leading help") {
-    val output = TestHelper.testMainException[ExitException](
+    val testOutput = TestHelper.testMainException[ExitException](
       "config", "view",
       "--help"
     )
 
-    assert(output.startsWith("viash config view"))
-    assert(!output.contains("testbash"))
+    assert(testOutput.stdout.startsWith("viash config view"))
+    assert(!testOutput.stdout.contains("testbash"))
   }
 
   test("viash config view default functionality trailing help") {
-    val output = TestHelper.testMainException[ExitException](
+    val testOutput = TestHelper.testMainException[ExitException](
       "config", "view",
       configFile,
       "--help"
     )
 
-    assert(output.startsWith("viash config view"))
-    assert(!output.contains("testbash"))
+    assert(testOutput.stdout.startsWith("viash config view"))
+    assert(!testOutput.stdout.contains("testbash"))
   }
 
   test("viash config view default functionality trailing help after platform argument") {
-    val output = TestHelper.testMainException[ExitException](
+    val testOutput = TestHelper.testMainException[ExitException](
       "config", "view",
       configFile,
-      "--platform", "native",
+      "--runner", "native",
       "--help"
     )
 
-    assert(output.startsWith("viash config view"))
-    assert(!output.contains("testbash"))
+    assert(testOutput.stdout.startsWith("viash config view"))
+    assert(!testOutput.stdout.contains("testbash"))
   }
 
   test("viash config view default functionality trailing help before platform argument") {
-    val output = TestHelper.testMainException[ExitException](
+    val testOutput = TestHelper.testMainException[ExitException](
       "config", "view",
       configFile,
       "--help",
-      "--platform", "native"
+      "--runner", "native"
     )
 
-    assert(output.startsWith("viash config view"))
-    assert(!output.contains("testbash"))
+    assert(testOutput.stdout.startsWith("viash config view"))
+    assert(!testOutput.stdout.contains("testbash"))
   }
 
 
   test("viash config view default functionality with --help as runner argument") {
-    val output = TestHelper.testMainException[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "config", "view",
       configFile,
       "--runner", "--help"
     )
 
-    assert(!output.contains("viash config view"))
+    assert(!testOutput.stdout.contains("viash config view"))
   }
 
 }
diff --git a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala
index 17a50ba01..84911f228 100644
--- a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala
@@ -34,7 +34,7 @@ class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{
 
   // convert testbash
   test("viash ns can build") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
     "ns", "build",
       "-s", nsPath,
       "-t", tempFolStr
@@ -42,7 +42,7 @@ class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{
 
     assert(nsFolder.exists)
     assert(nsFolder.isDirectory)
-    assert(exitCode == 1)
+    assert(testOutput.exitCode == Some(1))
 
     for ((component, _, _, _) <- components) {
       val executable = componentExecutableFile(component)
@@ -51,7 +51,7 @@ class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{
     }
 
     val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r
-    assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
+    assert(regexBuildError.findFirstIn(testOutput.stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
   }
 
   test("Check whether the executable can run") {
diff --git a/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala
index 946606b6c..21f0263ea 100644
--- a/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala
@@ -27,16 +27,15 @@ class MainNSExecNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
   private val tempFolStr = temporaryFolder.toString
 
   test("Check whether ns exec \\; works") {
-    val (stdoutRaw, stderrRaw, _) =
-      TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
         "ns", "exec",
         "--src", nsPath,
         "--apply_runner",
         "--apply_engine",
         "echo _{functionality-name}_ -{dir}- !{path}! ~{engine}~ ={namespace}=+\\;"
       )
-    val stdout = stdoutRaw.replaceAll(nsPath, "src/")
-    val stderr = stderrRaw.replaceAll(nsPath, "src/")
+    val stdout = testOutput.stdout.replaceAll(nsPath, "src/")
+    val stderr = testOutput.stderr.replaceAll(nsPath, "src/")
 
     for (component <- components) {
       val regexCommand = s"""\\+ echo _${component}_ -src/$component/?- !src/$component/config.vsh.yaml! ~native~ =testns=""".r
@@ -47,13 +46,13 @@ class MainNSExecNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
   }
 
   test("Check whether ns exec + works") {
-    val (stdoutRaw, stderrRaw, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "exec",
       "--src", nsPath,
       "echo {path} +"
     )
-    val stdout = stdoutRaw.replaceAll(nsPath, "src/")
-    val stderr = stderrRaw.replaceAll(nsPath, "src/")
+    val stdout = testOutput.stdout.replaceAll(nsPath, "src/")
+    val stderr = testOutput.stderr.replaceAll(nsPath, "src/")
 
     // can't guarantee order of components
     val regexCommand = s"""\\+ echo src/[^/]*/config.vsh.yaml src/[^/]*/config.vsh.yaml src/[^/]*/config.vsh.yaml src/[^/]*/config.vsh.yaml""".r
diff --git a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala
index 5afbeeb7b..47a1b49cd 100644
--- a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala
@@ -29,22 +29,22 @@ class MainNSListNativeSuite extends AnyFunSuite{
 
   // convert testbash
   test("viash ns list") {
-   val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+   val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
     )
 
-    assert(exitCode == 1)
+    assert(testOutput.exitCode == Some(1))
 
     for (component <- components) {
       val regexName = raw"""name:\s+"$component""""
-      assert(regexName.r.findFirstIn(stdout).isDefined, s"\nRegex: ${regexName}; text: \n$stdout")
+      assert(regexName.r.findFirstIn(testOutput.stdout).isDefined, s"\nRegex: ${regexName}; text: \n${testOutput.stdout}")
     }
 
     val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed"
-    assert(regexBuildError.r.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
+    assert(regexBuildError.r.findFirstIn(testOutput.stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
 
-    val stdout2 = s"(?s)(\u001b.{4})?((Not all configs parsed successfully)|(All \\d+ configs parsed successfully)).*$$".r.replaceAllIn(stdout, "")
+    val stdout2 = s"(?s)(\u001b.{4})?((Not all configs parsed successfully)|(All \\d+ configs parsed successfully)).*$$".r.replaceAllIn(testOutput.stdout, "")
 
     val config = parser.parse(stdout2)
       .fold(throw _, _.as[Array[Config]])
@@ -59,7 +59,7 @@ class MainNSListNativeSuite extends AnyFunSuite{
     )
 
     for ((regex, count) <- samples) {
-      assert(regex.r.findAllMatchIn(stdout).size == count, s"Expecting $count hits on stdout of regex [$regex]")
+      assert(regex.r.findAllMatchIn(testOutput.stdout).size == count, s"Expecting $count hits on stdout of regex [$regex]")
     }
   }
 
@@ -67,44 +67,44 @@ class MainNSListNativeSuite extends AnyFunSuite{
 
   // convert testbash
   test("viash ns list filter by engine and runner") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "--engine", "docker",
       "--runner", "docker"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 0)
   }
   test("viash ns list filter by engine #2") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", scalaPath,
       "--engine", "docker",
       "--runner", "executable"
     )
 
-    assert(exitCode == 0)
+    assert(testOutput.exitCode == Some(0))
 
-    val configs = parser.parse(stdout)
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
   }
   test("viash ns list filter by engine #3") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
      "ns", "list",
       "-s", scalaPath,
       "--engine", "not_exists",
       "--runner", "not_exists"
     )
 
-    assert(exitCode == 0)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(0))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 0)
@@ -112,200 +112,200 @@ class MainNSListNativeSuite extends AnyFunSuite{
 
   // test query_name
   test("viash ns list query_name") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
      "ns", "list",
       "-s", nsPath,
       "--query_name", "ns_add"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query_name full match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
      "ns", "list",
       "-s", nsPath,
       "--query_name", "^ns_add$"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query_name partial match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "--query_name", "add"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query_name no match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "--query_name", "foo"
     )
 
-    assert(exitCode == 1)
-    assert(stdout.trim() == "[]")
+    assert(testOutput.exitCode == Some(1))
+    assert(testOutput.stdout.trim() == "[]")
   }
 
   // test query
   test("viash ns list query") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "-q", "testns/ns_add"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query full match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "-q", "^testns/ns_add$"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query partial match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "-q", "test.*/.*add"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query only partial name") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "-q", "add"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
     assert(configs.length == 1)
-    assert(stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
   }
 
   test("viash ns list query no match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "-q", "foo"
     )
 
-    assert(exitCode == 1)
-    assert(stdout.trim() == "[]")
+    assert(testOutput.exitCode == Some(1))
+    assert(testOutput.stdout.trim() == "[]")
   }
 
   // test query_namespace
   test("viash ns list query_namespace") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
      "ns", "list",
       "-s", nsPath,
       "--query_namespace", "testns"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
 
     assert(configs.length == components.length)
-    assert(stdout.contains("name: \"ns_add\""))
-    assert(stdout.contains("name: \"ns_subtract\""))
-    assert(!stdout.contains("name: \"ns_error\""))
-    assert(!stdout.contains("name: \"ns_disabled\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_subtract\""))
+    assert(!testOutput.stdout.contains("name: \"ns_error\""))
+    assert(!testOutput.stdout.contains("name: \"ns_disabled\""))
   }
 
   test("viash ns list query_namespace full match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
      "ns", "list",
       "-s", nsPath,
       "--query_namespace", "^testns$"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
 
     assert(configs.length == components.length)
-    assert(stdout.contains("name: \"ns_add\""))
-    assert(stdout.contains("name: \"ns_subtract\""))
-    assert(!stdout.contains("name: \"ns_error\""))
-    assert(!stdout.contains("name: \"ns_disabled\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_subtract\""))
+    assert(!testOutput.stdout.contains("name: \"ns_error\""))
+    assert(!testOutput.stdout.contains("name: \"ns_disabled\""))
   }
 
   test("viash ns list query_namespace partial match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "--query_namespace", "test"
     )
 
-    assert(exitCode == 1)
-    val configs = parser.parse(stdout)
+    assert(testOutput.exitCode == Some(1))
+    val configs = parser.parse(testOutput.stdout)
       .fold(throw _, _.as[Array[Config]])
       .fold(throw _, identity)
 
     assert(configs.length == components.length)
-    assert(stdout.contains("name: \"ns_add\""))
-    assert(stdout.contains("name: \"ns_subtract\""))
-    assert(!stdout.contains("name: \"ns_error\""))
-    assert(!stdout.contains("name: \"ns_disabled\""))
+    assert(testOutput.stdout.contains("name: \"ns_add\""))
+    assert(testOutput.stdout.contains("name: \"ns_subtract\""))
+    assert(!testOutput.stdout.contains("name: \"ns_error\""))
+    assert(!testOutput.stdout.contains("name: \"ns_disabled\""))
   }
 
   test("viash ns list query_namespace no match") {
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "list",
       "-s", nsPath,
       "--query_namespace", "foo"
     )
 
-    assert(exitCode == 1)
-    assert(stdout.trim() == "[]")
+    assert(testOutput.exitCode == Some(1))
+    assert(testOutput.stdout.trim() == "[]")
   }
 
 }
diff --git a/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala
index dee8976ea..add2216bc 100644
--- a/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala
@@ -47,7 +47,7 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
 
 
   test("Check namespace test output without working dir message") {
-    val (stdout, stderr, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "test",
       "--src", nsPath,
       "--keep", "false"
@@ -55,24 +55,24 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
 
     // Test inclusion of a header
     val regexHeader = raw"^\s*namespace\s*functionality\s*runner\s*engine\s*test_name\s*exit_code\s*duration\s*result".r
-    assert(regexHeader.findFirstIn(stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n$stdout")
+    assert(regexHeader.findFirstIn(testOutput.stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${testOutput.stdout}")
 
     for (
       (component, steps) <- components;
       (step, resultPattern) <- steps
     ) {
       val regex = s"""testns\\s*$component\\s*executable\\s*native\\s*$step$resultPattern""".r
-      assert(regex.findFirstIn(stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n$stdout")
+      assert(regex.findFirstIn(testOutput.stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n${testOutput.stdout}")
     }
 
     val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r
-    assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
+    assert(regexBuildError.findFirstIn(testOutput.stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
 
-    assert(stderr.contains("The status of the component 'ns_power' is set to deprecated."))
+    assert(testOutput.stderr.contains("The status of the component 'ns_power' is set to deprecated."))
   }
 
   test("Check namespace test output with working dir message") {
-    val (stdout, stderr, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "test",
       "--src", nsPath,
       "--keep", "true"
@@ -80,27 +80,27 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
 
     // Test inclusion of a header
     val regexHeader = raw"^\s*namespace\s*functionality\s*runner\s*engine\s*test_name\s*exit_code\s*duration\s*result".r
-    assert(regexHeader.findFirstIn(stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n$stdout")
+    assert(regexHeader.findFirstIn(testOutput.stdout).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${testOutput.stdout}")
 
     val regexWdir = raw"The working directory for the namespace tests is [\w/]+[\r\n]{1,2}".r
-    assert(regexWdir.findFirstIn(stderr).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n$stderr")
+    assert(regexWdir.findFirstIn(testOutput.stderr).isDefined, s"\nRegex: ${regexHeader.toString}; text: \n${testOutput.stderr}")
 
     for (
       (component, steps) <- components;
       (step, resultPattern) <- steps
     ) {
       val regex = s"""testns\\s*$component\\s*executable\\s*native\\s*$step$resultPattern""".r
-      assert(regex.findFirstIn(stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n$stdout")
+      assert(regex.findFirstIn(testOutput.stdout).isDefined, s"\nRegex: '${regex.toString}'; text: \n${testOutput.stdout}")
     }
 
     val regexBuildError = raw"Reading file \'.*/src/ns_error/config\.vsh\.yaml\' failed".r
-    assert(regexBuildError.findFirstIn(stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
+    assert(regexBuildError.findFirstIn(testOutput.stderr).isDefined, "Expecting to get an error because of an invalid yaml in ns_error")
   }
 
   test("Check namespace test output with tsv option") {
     val log = Paths.get(tempFolStr, "log.tsv").toFile
 
-    val (testText, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "ns", "test",
       "--tsv", log.toString,
       "--src", nsPath
@@ -133,7 +133,7 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
     val fileHeader = "Test header" + sys.props("line.separator")
     Files.write(log.toPath, fileHeader.getBytes(StandardCharsets.UTF_8))
 
-    val (testText, _, _) = TestHelper.testMainWithStdErr(
+    TestHelper.testMain(
       "ns", "test",
       "--tsv", log.toString,
       "--append",
@@ -166,7 +166,7 @@ class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
   test("Check namespace test output with tsv and append options without the output file exists") {
     val log = Paths.get(tempFolStr, "log_append_new.tsv").toFile
 
-    val (testText, _, _) = TestHelper.testMainWithStdErr(
+    TestHelper.testMain(
       "ns", "test",
       "--tsv", log.toString,
       "--append",
diff --git a/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala b/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala
index 768ca2dbf..6e01480f8 100644
--- a/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala
+++ b/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala
@@ -15,105 +15,105 @@ class RunComputationalRequirements extends AnyFunSuite with BeforeAndAfterAll {
   private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryFolder)
   
   test("Check without computational requirements") {
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       configFile
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set cpus in CLI") {
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       "--cpus", "2",
       configFile
     )
 
-    assert(output.contains("cpus: 2"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus: 2"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set memory in CLI") {
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       "--memory", "2mb",
       configFile
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory: 2"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory: 2"))
   }
 
   test("Check set cpus in config") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3}""", "cpus_set")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 3"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus: 3"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set memory in config") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {memory: "3 mb"}""", "memory_set")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory: 3"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory: 3"))
   }
 
   test("Check set cpus and memory in config") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3, memory: "3 mb"}""", "cpus_memory_set")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 3"))
-    assert(output.contains("memory: 3"))
+    assert(testOutput.stdout.contains("cpus: 3"))
+    assert(testOutput.stdout.contains("memory: 3"))
   }
 
   test("Check set cpus in config and CLI") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3}""", "cpus_set2")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       "--cpus", "2",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 2"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus: 2"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set memory in config and CLI") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {memory: "3 mb"}""", "memory_set2")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       "--memory", "2mb",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory: 2"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory: 2"))
   }
 
   test("Check set cpus and memory in config and CLI") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3, memory: "3 mb"}""", "cpus_memory_set2")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "run",
       "--cpus", "2",
       "--memory", "2mb",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 2"))
-    assert(output.contains("memory: 2"))
+    assert(testOutput.stdout.contains("cpus: 2"))
+    assert(testOutput.stdout.contains("memory: 2"))
   }
 
   override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala
index 92c85ec00..e575be536 100644
--- a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala
+++ b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala
@@ -22,37 +22,37 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
   private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryFolder)
 
   test("Check standard test output for typical outputs", DockerTest) {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "--runner", "docker",
       configFile
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check standard test output with trailing arguments", DockerTest) {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       configFile,
       "--engine", "docker",
       "--runner", "docker"
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check standard test output with leading and trailing arguments", DockerTest) {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "--runner", "docker",
@@ -60,17 +60,17 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
       "-k", "false"
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check setup strategy", DockerTest) {
     val newConfigFilePath = configDeriver.derive(""".engines[.type == "docker" && !has(.id) ].setup := [{ type: "docker", run: "echo 'Hello world!'" }]""", "cache_config")
     // first run to create cache entries
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "--runner", "docker",
@@ -79,7 +79,7 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
     )
 
     // Do a second run to check if forcing a docker build using setup works
-    val testTextNoCaching = TestHelper.testMain(
+    val testOutputNoCaching = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "--runner", "docker",
@@ -89,10 +89,10 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
     )
 
     val regexBuildCache = raw"\n#\d \[\d/\d\] RUN echo 'Hello world!'\n#\d CACHED\n".r
-    assert(!regexBuildCache.findFirstIn(testTextNoCaching).isDefined, "Expected to not find caching.")
+    assert(!regexBuildCache.findFirstIn(testOutputNoCaching.stdout).isDefined, "Expected to not find caching.")
 
     // Do a third run to check caching
-    val testTextCaching = TestHelper.testMain(
+    val testOutputCaching = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "--runner", "docker",
@@ -103,10 +103,10 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
 
     // retry once if it failed
     val testTextCachingWithRetry = 
-      if (regexBuildCache.findFirstIn(testTextCaching).isDefined) {
-        testTextCaching
+      if (regexBuildCache.findFirstIn(testOutputCaching.stdout).isDefined) {
+        testOutputCaching
       } else {
-        checkTempDirAndRemove(testTextCaching, false)
+        checkTempDirAndRemove(testOutputCaching.stdout, false)
         
         TestHelper.testMain(
           "test",
@@ -118,62 +118,62 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
         )
       }
 
-    assert(regexBuildCache.findFirstIn(testTextCachingWithRetry).isDefined, "Expected to find caching.")
+    assert(regexBuildCache.findFirstIn(testTextCachingWithRetry.stdout).isDefined, "Expected to find caching.")
 
-    checkTempDirAndRemove(testText, false)
-    checkTempDirAndRemove(testTextNoCaching, false)
-    checkTempDirAndRemove(testTextCachingWithRetry, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
+    checkTempDirAndRemove(testOutputNoCaching.stdout, false)
+    checkTempDirAndRemove(testTextCachingWithRetry.stdout, false)
   }
 
   test("Verify base config derivation", NativeTest) {
     val newConfigFilePath = configDeriver.derive(Nil, "default_config")
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check failing build", DockerTest) {
     val newConfigFilePath = configDeriver.derive(""".engines[.type == "docker" && !has(.id) ].setup := [{ type: "apt", packages: ["get_the_machine_that_goes_ping"] }]""", "failed_build")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "docker",
       "--runner", "docker",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText == "Setup failed!")
+    assert(testOutput.exceptionText.get == "Setup failed!")
 
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(testOutput.output.contains("ERROR! Setup failed!"))
-    assert(!testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Setup failed!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testOutput.output, true)
+    checkTempDirAndRemove(testOutput.stdout, true)
   }
 
   test("Check config and resource files with spaces in the filename", DockerTest) {
     val newConfigFilePath = Paths.get(tempFolStr, "config with spaces.vsh.yaml")
     Files.copy(Paths.get(configFile), newConfigFilePath)
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "--runner", "docker",
       newConfigFilePath.toString()
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   /**
diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala
index 075d8a507..423456ad3 100644
--- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala
@@ -10,6 +10,8 @@ import org.scalatest.funsuite.AnyFunSuite
 
 import scala.reflect.io.Directory
 import sys.process._
+import io.viash.exceptions.ConfigParserException
+import io.viash.exceptions.MissingResourceFileException
 
 class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
   Logger.UseColorOverride.value = Some(false)
@@ -23,37 +25,37 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
   private val configInvalidYamlFile = getClass.getResource("/testbash/invalid_configs/config_invalid_yaml.vsh.yaml").getPath
 
   test("Check standard test output for typical outputs") {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
       configFile
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check standard test output with trailing arguments") {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       configFile,
       "--engine", "native",
       "--runner", "native"
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check standard test output with leading and trailing arguments") {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
@@ -61,243 +63,243 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       "-k", "false"
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Verify base config derivation") {
     val newConfigFilePath = configDeriver.derive(Nil, "default_config")
 
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check test output when no tests are specified in the functionality file") {
     val newConfigFilePath = configDeriver.derive("del(.functionality.test_resources)", "no_tests")
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("WARNING! No tests found!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("WARNING! No tests found!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check test output when a test fails") {
     val newConfigFilePath = configDeriver.derive(""".functionality.test_resources[.path == "tests/check_outputs.sh"].path := "tests/fail_failed_test.sh"""", "failed_test")
-    val testText = TestHelper.testMainException[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
-    assert(!testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, true)
+    checkTempDirAndRemove(testOutput.stdout, true)
   }
 
   test("Check test output when test doesn't exist") {
     val newConfigFilePath = configDeriver.derive(""".functionality.test_resources[.path == "tests/check_outputs.sh"].path := "tests/nonexistent_test.sh"""", "nonexisting_test")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText == "Only 1 out of 2 test scripts succeeded!")
+    assert(testOutput.exceptionText.get == "Only 1 out of 2 test scripts succeeded!")
 
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(testOutput.output.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
-    assert(!testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testOutput.output, true)
+    checkTempDirAndRemove(testOutput.stdout, true)
   }
 
   test("Check config and resource files with spaces in the filename") {
     val newConfigFilePath = Paths.get(tempFolStr, "config with spaces.vsh.yaml")
     Files.copy(Paths.get(configFile), newConfigFilePath)
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath.toString()
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check config file without 'functionality' specified") {
     val newConfigFilePath = configDeriver.derive("""del(.functionality)""", "missing_functionality")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("must be a yaml file containing a viash config."))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.exceptionText.get.contains("must be a yaml file containing a viash config."))
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid runner type") {
     val newConfigFilePath = configDeriver.derive(""".runners += { type: "foo" }""", "invalid_runner_type")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("Type 'foo' is not recognised. Valid types are 'executable', and 'nextflow'."))
-    assert(testOutput.exceptionText.contains(
+    assert(testOutput.exceptionText.get.contains("Type 'foo' is not recognised. Valid types are 'executable', and 'nextflow'."))
+    assert(testOutput.exceptionText.get.contains(
       """{
         |  "type" : "foo"
         |}""".stripMargin))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid engine type") {
     val newConfigFilePath = configDeriver.derive(""".engines += { type: "foo" }""", "invalid_engine_type")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("Type 'foo' is not recognised. Valid types are 'docker', and 'native'."))
-    assert(testOutput.exceptionText.contains(
+    assert(testOutput.exceptionText.get.contains("Type 'foo' is not recognised. Valid types are 'docker', and 'native'."))
+    assert(testOutput.exceptionText.get.contains(
       """{
         |  "type" : "foo"
         |}""".stripMargin))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid platform type") {
     val newConfigFilePath = configDeriver.derive(""".platforms := [{ type: "foo" }]""", "invalid_platform_type")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("Type 'foo' is not recognised. Valid types are 'docker', 'native', and 'nextflow'."))
-    assert(testOutput.exceptionText.contains(
+    assert(testOutput.exceptionText.get.contains("Type 'foo' is not recognised. Valid types are 'docker', 'native', and 'nextflow'."))
+    assert(testOutput.exceptionText.get.contains(
       """{
         |  "type" : "foo"
         |}""".stripMargin))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid field in runner") {
     val newConfigFilePath = configDeriver.derive(""".runners += { type: "executable", foo: "bar" }""", "invalid_runner_field")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("Invalid data fields for ExecutableRunner."))
-    assert(testOutput.exceptionText.contains(
+    assert(testOutput.exceptionText.get.contains("Invalid data fields for ExecutableRunner."))
+    assert(testOutput.exceptionText.get.contains(
       """{
         |  "type" : "executable",
         |  "foo" : "bar"
         |}""".stripMargin))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid field in engine") {
     val newConfigFilePath = configDeriver.derive(""".engines += { type: "native", foo: "bar" }""", "invalid_engine_field")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("Invalid data fields for NativeEngine."))
-    assert(testOutput.exceptionText.contains(
+    assert(testOutput.exceptionText.get.contains("Invalid data fields for NativeEngine."))
+    assert(testOutput.exceptionText.get.contains(
       """{
         |  "type" : "native",
         |  "foo" : "bar"
         |}""".stripMargin))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid field in platform") {
     val newConfigFilePath = configDeriver.derive(""".platforms := [{ type: "native", foo: "bar" }]""", "invalid_platform_field")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[ConfigParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.contains("Invalid data fields for NativePlatform."))
-    assert(testOutput.exceptionText.contains(
+    assert(testOutput.exceptionText.get.contains("Invalid data fields for NativePlatform."))
+    assert(testOutput.exceptionText.get.contains(
       """{
         |  "type" : "native",
         |  "foo" : "bar"
         |}""".stripMargin))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check valid viash config yaml but with wrong file extension") {
     val newConfigFilePath = Paths.get(tempFolStr, "config.txt")
     Files.copy(Paths.get(configFile), newConfigFilePath)
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath.toString()
     )
 
-    assert(testOutput.exceptionText.contains("must be a yaml file containing a viash config."))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.exceptionText.get.contains("must be a yaml file containing a viash config."))
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check invalid viash config yaml") {
-    val testOutput = TestHelper.testMainException2[io.circe.ParsingFailure](
+    val testOutput = TestHelper.testMainException[org.yaml.snakeyaml.parser.ParserException](
       "test",
       "--engine", "native",
       "--runner", "native",
       configInvalidYamlFile
     )
 
-    assert(testOutput.exceptionText.contains("while parsing a flow mapping"))
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.exceptionText.get.contains("while parsing a flow mapping"))
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check output in case --keep true is specified") {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
@@ -305,15 +307,15 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       configFile
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(!testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, true)
+    checkTempDirAndRemove(testOutput.stdout, true)
   }
 
   test("Check output in case --keep false is specified") {
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
@@ -321,16 +323,16 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       configFile
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check test output when a test fails and --keep true is specified") {
     val newConfigFilePath = configDeriver.derive(""".functionality.test_resources[.path == "tests/check_outputs.sh"].path := "tests/fail_failed_test.sh"""", "failed_test_keep_true")
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "native",
@@ -338,13 +340,13 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText == "Only 1 out of 2 test scripts succeeded!")
+    assert(testOutput.exceptionText.get == "Only 1 out of 2 test scripts succeeded!")
 
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(testOutput.output.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
-    assert(!testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testOutput.output, true)
+    checkTempDirAndRemove(testOutput.stdout, true)
   }
 
   test("Check test output when a test fails and --keep false is specified") {
@@ -352,7 +354,7 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       """.functionality.test_resources[.path == "tests/check_outputs.sh"].path := "tests/fail_failed_test.sh"""",
       "failed_test_keep_false"
     )
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "native",
@@ -360,69 +362,69 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText == "Only 1 out of 2 test scripts succeeded!")
+    assert(testOutput.exceptionText.get == "Only 1 out of 2 test scripts succeeded!")
 
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(testOutput.output.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
-    assert(testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Only 1 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testOutput.output, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check deprecation warning") {
     val newConfigFilePath = configDeriver.derive(""".functionality.status := "deprecated"""", "deprecated")
-    val (testText, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(exitCode == 0)
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.exitCode == Some(0))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 2 out of 2 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    assert(stderr.contains("The status of the component 'testbash' is set to deprecated."))
+    assert(testOutput.stderr.contains("The status of the component 'testbash' is set to deprecated."))
     
-    checkTempDirAndRemove(testText, false)
+    checkTempDirAndRemove(testOutput.stdout, false)
   }
 
   test("Check standard test output with bad engine name") {
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "non_existing_engine",
       "--runner", "native",
       configFile
     )
 
-    assert(testOutput.exceptionText == "no engine id matching regex 'non_existing_engine' could not be found in the config.")
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.exceptionText.get == "no engine id matching regex 'non_existing_engine' could not be found in the config.")
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check standard test output with bad runner name") {
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "native",
       "--runner", "non_existing_runner",
       configFile
     )
 
-    assert(testOutput.exceptionText == "no runner id matching regex 'non_existing_runner' could not be found in the config.")
-    assert(testOutput.output.isEmpty)
+    assert(testOutput.exceptionText.get == "no runner id matching regex 'non_existing_runner' could not be found in the config.")
+    assert(testOutput.stdout.isEmpty)
   }
 
   test("Check standard test output with missing test resource") {
     val newConfigFilePath = configDeriver.derive(""".functionality.test_resources += { type: 'file', path: 'foobar.txt' }""", "deprecated")
 
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[MissingResourceFileException](
       "test",
       "--engine", "native",
       "--runner", "native",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText.matches("Missing resource .*foobar\\.txt as specified in .*"))
+    assert(testOutput.exceptionText.get.matches("Missing resource .*foobar\\.txt as specified in .*"))
   }
 
   /**
diff --git a/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala b/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala
index b8a7f48cc..85c25d6fb 100644
--- a/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala
+++ b/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala
@@ -15,105 +15,105 @@ class TestComputationalRequirements extends AnyFunSuite with BeforeAndAfterAll {
   private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryFolder)
   
   test("Check without computational requirements") {
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       configFile
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set cpus in CLI") {
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--cpus", "2",
       configFile
     )
 
-    assert(output.contains("cpus: 2"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus: 2"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set memory in CLI") {
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--memory", "2mb",
       configFile
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory: 2"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory: 2"))
   }
 
   test("Check set cpus in config") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3}""", "cpus_set")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 3"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus: 3"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set memory in config") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {memory: "3 mb"}""", "memory_set")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory: 3"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory: 3"))
   }
 
   test("Check set cpus and memory in config") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3, memory: "3 mb"}""", "cpus_memory_set")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 3"))
-    assert(output.contains("memory: 3"))
+    assert(testOutput.stdout.contains("cpus: 3"))
+    assert(testOutput.stdout.contains("memory: 3"))
   }
 
   test("Check set cpus in config and CLI") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3}""", "cpus_set2")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--cpus", "2",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 2"))
-    assert(output.contains("memory unset"))
+    assert(testOutput.stdout.contains("cpus: 2"))
+    assert(testOutput.stdout.contains("memory unset"))
   }
 
   test("Check set memory in config and CLI") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {memory: "3 mb"}""", "memory_set2")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--memory", "2mb",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus unset"))
-    assert(output.contains("memory: 2"))
+    assert(testOutput.stdout.contains("cpus unset"))
+    assert(testOutput.stdout.contains("memory: 2"))
   }
 
   test("Check set cpus and memory in config and CLI") {
     val newConfigFilePath = configDeriver.derive(""".functionality.requirements := {cpus: 3, memory: "3 mb"}""", "cpus_memory_set2")
-    val output = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--cpus", "2",
       "--memory", "2mb",
       newConfigFilePath
     )
 
-    assert(output.contains("cpus: 2"))
-    assert(output.contains("memory: 2"))
+    assert(testOutput.stdout.contains("cpus: 2"))
+    assert(testOutput.stdout.contains("memory: 2"))
   }
 
   override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/functionality/dependencies/Dependency.scala b/src/test/scala/io/viash/functionality/dependencies/Dependency.scala
index bb1c9fe1c..12199bceb 100644
--- a/src/test/scala/io/viash/functionality/dependencies/Dependency.scala
+++ b/src/test/scala/io/viash/functionality/dependencies/Dependency.scala
@@ -56,13 +56,123 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll {
     writeTestConfig(testFolder.resolve("src/dep2/config.vsh.yaml"), fun2)
 
     // build
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
         "ns", "build",
         "-s", testFolder.resolve("src").toString(),
         "-t", testFolder.resolve("target").toString()
       )
 
-    assert(stderr.strip == "All 2 configs built successfully", "check build was successful")
+    assert(testOutput.stderr.strip == "All 2 configs built successfully", "check build was successful")
+
+    // check file & file content
+    val outputPath = testFolder.resolve("target/executable/dep2/dep2")
+    val executable = outputPath.toFile
+    assert(executable.exists)
+    assert(executable.canExecute)
+
+    val outputText = IO.read(outputPath.toUri())
+    assert(outputText.contains("VIASH_DEP_DEP1="), "check the dependency is set in the output script")
+
+    // check output when running
+    val out = Exec.runCatch(
+      Seq(executable.toString)
+    )
+
+    assert(out.output == "Hello from dep1\nHello from dep2\n")
+    assert(out.exitValue == 0)
+  }
+
+  test("Use a local repository with an absolute path") {
+    val testFolder = createViashSubFolder(temporaryFolder, "local_test_absolute_path")
+    
+    // write test files
+    val fun1 = Functionality(
+      name = "dep1",
+      resources = textBashScript("echo Hello from dep1"),
+    )
+    val fun2 = Functionality(
+      name = "dep2",
+      resources = textBashScript("$dep_dep1\necho Hello from dep2"),
+      dependencies = List(Dependency("dep1", repository = Right(LocalRepository(path = Some("/dependencies")))))
+    )
+
+    writeTestConfig(testFolder.resolve("dependencies/src/dep1/config.vsh.yaml"), fun1)
+    writeTestConfig(testFolder.resolve("src/dep2/config.vsh.yaml"), fun2)
+
+    // build our local repository
+    val build1 = TestHelper.testMain(
+        workingDir = Some(testFolder.resolve("dependencies")),
+        "ns", "build",
+        "-s", testFolder.resolve("dependencies/src").toString(),
+        "-t", testFolder.resolve("dependencies/target").toString()
+      )
+    
+    assert(build1.stderr.strip == "All 1 configs built successfully", "check dependency build was successful")
+
+    // build
+    val build2 = TestHelper.testMain(
+        workingDir = Some(testFolder),
+        "ns", "build",
+        "-s", testFolder.resolve("src").toString(),
+        "-t", testFolder.resolve("target").toString()
+      )
+
+    assert(build2.stderr.strip == "All 1 configs built successfully", "check build was successful")
+
+    // check file & file content
+    val outputPath = testFolder.resolve("target/executable/dep2/dep2")
+    val executable = outputPath.toFile
+    assert(executable.exists)
+    assert(executable.canExecute)
+
+    val outputText = IO.read(outputPath.toUri())
+    assert(outputText.contains("VIASH_DEP_DEP1="), "check the dependency is set in the output script")
+
+    // check output when running
+    val out = Exec.runCatch(
+      Seq(executable.toString)
+    )
+
+    assert(out.output == "Hello from dep1\nHello from dep2\n")
+    assert(out.exitValue == 0)
+  }
+
+  test("Use a local repository with a relative path") {
+    val testFolder = createViashSubFolder(temporaryFolder, "local_test_absolute_path")
+    
+    // write test files
+    val fun1 = Functionality(
+      name = "dep1",
+      resources = textBashScript("echo Hello from dep1"),
+    )
+    val fun2 = Functionality(
+      name = "dep2",
+      resources = textBashScript("$dep_dep1\necho Hello from dep2"),
+      dependencies = List(Dependency("dep1", repository = Right(LocalRepository(path = Some("../../dependencies")))))
+    )
+
+    writeTestConfig(testFolder.resolve("dependencies/src/dep1/config.vsh.yaml"), fun1)
+    writeTestConfig(testFolder.resolve("src/dep2/config.vsh.yaml"), fun2)
+
+    // build our local repository
+    val build1 = TestHelper.testMain(
+        workingDir = Some(testFolder.resolve("dependencies")),
+        "ns", "build",
+        "-s", testFolder.resolve("dependencies/src").toString(),
+        "-t", testFolder.resolve("dependencies/target").toString()
+      )
+    
+    assert(build1.stderr.strip == "All 1 configs built successfully", "check dependency build was successful")
+
+    // build
+    val build2 = TestHelper.testMain(
+        workingDir = Some(testFolder),
+        "ns", "build",
+        "-s", testFolder.resolve("src").toString(),
+        "-t", testFolder.resolve("target").toString()
+      )
+
+    assert(build2.stderr.strip == "All 1 configs built successfully", "check build was successful")
 
     // check file & file content
     val outputPath = testFolder.resolve("target/executable/dep2/dep2")
@@ -95,13 +205,13 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll {
     writeTestConfig(testFolder.resolve("src/dep3/config.vsh.yaml"), fun)
 
     // build
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
         "ns", "build",
         "-s", testFolder.resolve("src").toString(),
         "-t", testFolder.resolve("target").toString()
       )
 
-    assert(stderr.strip.contains("All 1 configs built successfully"), "check build was successful")
+    assert(testOutput.stderr.strip.contains("All 1 configs built successfully"), "check build was successful")
 
     // check file & file content
     val outputPath = testFolder.resolve("target/executable/dep3/dep3")
@@ -134,13 +244,13 @@ class DependencyTest extends AnyFunSuite with BeforeAndAfterAll {
     writeTestConfig(testFolder.resolve("src/dep4/config.vsh.yaml"), fun)
 
     // build
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
         "ns", "build",
         "-s", testFolder.resolve("src").toString(),
         "-t", testFolder.resolve("target").toString()
       )
 
-    assert(stderr.strip.contains("All 1 configs built successfully"), "check build was successful")
+    assert(testOutput.stderr.strip.contains("All 1 configs built successfully"), "check build was successful")
 
     // check file & file content
     val outputPath = testFolder.resolve("target/executable/dep4/dep4")
diff --git a/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala b/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala
index 55f213787..8bb48fcbf 100644
--- a/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala
@@ -36,9 +36,7 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll {
 
   test("Build pipeline components", DockerTest, NextflowTest) {
     // build the nextflow containers
-    // TODO: use the correct CWD to build the pipeline to be ablke
-    // to detect the correct path to the _viash.yaml file
-    val (_, _, _) = TestHelper.testMainWithStdErr(
+    TestHelper.testMain(
       "ns", "build",
       "-s", srcPath,
       "-t", targetPath,
diff --git a/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala b/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala
index 92144e98d..c048348c7 100644
--- a/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala
@@ -35,7 +35,7 @@ class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll {
 
   test("Build pipeline components", DockerTest, NextflowTest) {
     // build the nextflow containers
-    val (_, _, _) = TestHelper.testMainWithStdErr(
+    TestHelper.testMain(
       "ns", "build",
       "--runner", "nextflow",
       "-s", srcPath,
@@ -101,15 +101,15 @@ class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll {
     val correctedStdOut2 = regex.matcher(correctedStdOut1).replaceAll("")
 
     // run Viash's --help
-    val (stdOut2, stdErr2, exitCode2) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "run", workflowsPath + "/pipeline3/config.vsh.yaml",
       "--", "--help"
     )
 
-    assert(exitCode2 == 0)
+    assert(testOutput.exitCode == Some(0))
 
     // check if they are the same
-    assert(correctedStdOut2 == stdOut2)
+    assert(correctedStdOut2 == testOutput.stdout)
   }
 
   override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala b/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala
index 93447eeb8..31260ac98 100644
--- a/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala
@@ -35,7 +35,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll {
 
   test("Build pipeline components", DockerTest, NextflowTest) {
     // build the nextflow containers
-    val (_, _, _) = TestHelper.testMainWithStdErr(
+    TestHelper.testMain(
       "ns", "build",
       "-s", srcPath,
       "-t", targetPath,
diff --git a/src/test/scala/io/viash/runners/nextflow/WorkflowHelperTest.scala b/src/test/scala/io/viash/runners/nextflow/WorkflowHelperTest.scala
index 8a12836b9..ad9a3e9c1 100644
--- a/src/test/scala/io/viash/runners/nextflow/WorkflowHelperTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/WorkflowHelperTest.scala
@@ -37,7 +37,7 @@ class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll {
 
   test("Build pipeline components", DockerTest, NextflowTest) {
     // build the nextflow containers
-    val (_, _, _) = TestHelper.testMainWithStdErr(
+    TestHelper.testMain(
       "ns", "build",
       "-s", srcPath,
       "-t", targetPath,