Feature/local dependency test2 (#565)

* Test local repositories Test with both absolute and relative paths. * Refactor TestHelper class Extend ExceptionOutput with exitCode and use it in all functions Remove testMain and replace it with testMainWithStdErr functionality and rename it testMain as the sole function Combine functionality of testMainException and testMainException2 * Fix modified testbench checking the wrong output for text * Fix cherry picking * Add changelog entry * Fix merge
viash-io · Oct 13, 2023 · 6afbc30 · 6afbc30
1 parent eddbdfe
commit 6afbc30
Show file tree

Hide file tree

Showing 24 changed files with 565 additions and 478 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,12 @@ TODO add summary
 * `export json_schema`: Add a `--strict` option to output a subset of the schema representing the internal structure of the Viash config (PR #564).
 
 * `config view` and `ns list`: Do not output internal functionality fields (#564). Additionally, add a validation that no internal fields are present when reading a Viash config file.
+
+## MINOR CHANGES
+
+* `testbenches`: Add testbenches for local dependencies (PR #565).
+
+* `testbenches`: Refactor testbenches helper functions to uniformize them (PR #565).
 
 # Viash 0.8.0-RC5 (2023-10-11): Fix run workflow
 

diff --git a/src/test/scala/io/viash/TestHelper.scala b/src/test/scala/io/viash/TestHelper.scala
@@ -10,94 +10,65 @@ import scala.reflect.ClassTag
 
 object TestHelper {
 
-  case class ExceptionOutput(
-    exceptionText: String,
-    output: String,
-    error: String,
+  case class TestMainOutput(
+    stdout: String,
+    stderr: String,
+    exitCode: Option[Int],
+    exceptionText: Option[String],
   )
 
   /**
-   * Method to capture the console stdout generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout being printed to the console, we print it after the Main.main() is finished.
+   * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console
    * @param args all the arguments typically passed to Main.main()
-   * @return a string of all the output
+   * @return TestMainOutput containing the console output text and exit code
    */
-  def testMain(args: String*) : String = {
-    val os = new ByteArrayOutputStream()
-    Console.withErr(os) {
-      Console.withOut(os) {
-        Main.mainCLI(args.toArray)
-      }
-    }
-
-    val stdout = os.toString
-    // Console.print(stdout)
-    stdout
-  }
+  def testMain(args: String*): TestMainOutput = testMain(None, args: _*)
 
   /**
    * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout and stderr being printed to the console, we print it after the Main.main() is finished.
+   * @param workingDir the working directory to run the command in
    * @param args all the arguments typically passed to Main.main()
-   * @return a tuple of stdout and stderr strings of all the output
+   * @return TestMainOutput containing the console output text and exit code
    */
-  def testMainWithStdErr(args: String*) : (String, String, Int) = {
+  def testMain(workingDir: Option[Path], args: String*): TestMainOutput = {
     val outStream = new ByteArrayOutputStream()
     val errStream = new ByteArrayOutputStream()
     val exitCode = Console.withOut(outStream) {
       Console.withErr(errStream) {
-        Main.mainCLI(args.toArray)
+        Main.mainCLI(args.toArray, workingDir)
       }
     }
 
-    val stdout = outStream.toString
-    val stderr = errStream.toString
-    // Console.print(stdout)
-    (stdout, stderr, exitCode)
+    TestMainOutput(outStream.toString, errStream.toString, Some(exitCode), None)
   }
 
   /**
-   * Method to capture the console stdout generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout being printed to the console, we print it after the Main.main() is finished.
-   * Additionally it handles a thrown RuntimeException using assertThrows
+   * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console.
+   * Additionally it handles a thrown Exception/Throwable and returns the exception text.
    * @param args all the arguments typically passed to Main.main()
-   * @return a string of all the output
+   * @return TestMainOutput containing the exception text and the console output text
    */
-  def testMainException[T <: AnyRef: ClassTag](args: String*) : String = {
-    val os = new ByteArrayOutputStream()
-    assertThrows[T] {
-      Console.withOut(os) {
-        Main.mainCLI(args.toArray)
-      }
-    }
-
-    val stdout = os.toString
-    // Console.print(stdout)
-    stdout
-  }
+  def testMainException[T <: Throwable](args: String*)(implicit classTag: ClassTag[T]) : TestMainOutput = testMainException(None, args: _*)
 
   /**
-   * Method to capture the console stdout generated by Main.main() so we can analyse what's being outputted to the console
-   * As the capture prevents the stdout being printed to the console, we print it after the Main.main() is finished.
-   * Additionally it handles a thrown RuntimeException using assertThrows
+   * Method to capture the console stdout and stderr generated by Main.main() so we can analyse what's being outputted to the console.
+   * Additionally it handles a thrown Exception/Throwable and returns the exception text.
+   * @param workingDir the working directory to run the command in
    * @param args all the arguments typically passed to Main.main()
-   * @return ExceptionOutput containing the exception text and the console output text
+   * @return TestMainOutput containing the exception text and the console output text
    */
-  def testMainException2[T <: Exception](args: String*) : ExceptionOutput = {
+  def testMainException[T <: Throwable](workingDir: Option[Path], args: String*)(implicit classTag: ClassTag[T]) : TestMainOutput = {
     val outStream = new ByteArrayOutputStream()
     val errStream = new ByteArrayOutputStream()
-    val caught = intercept[Exception] {
+    val caught = intercept[T] {
       Console.withOut(outStream) {
         Console.withErr(errStream) {
-          Main.mainCLI(args.toArray)
+          Main.mainCLI(args.toArray, workingDir)
         }
       }
     }
 
-    val stdout = outStream.toString
-    val stderr = errStream.toString
-    // Console.print(stdout)
-    ExceptionOutput(caught.getMessage, stdout, stderr)
+    TestMainOutput(outStream.toString, errStream.toString, None, Some(caught.getMessage))
   }
 
 

diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala
@@ -116,13 +116,13 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll w
   test("Test with platform and chown is set to false", DockerTest) {
     val newConfigFile = configDeriver.derive(""".platforms := [{type: "docker", chown: false }]""", "docker_chown_false")
     // functionality not provided in runner, should throw exception
-    val output = TestHelper.testMainException2[ConfigParserException](
+    val output = TestHelper.testMainException[ConfigParserException](
       "build",
       "--engine", "docker_chown",
       "-o", tempFolStr,
       newConfigFile
     )
-    assert(output.error.contains("Error: ..chown was removed: Compability not provided with the Runners functionality."))
+    assert(output.stderr.contains("Error: ..chown was removed: Compability not provided with the Runners functionality."))
   }
 
   override def afterAll(): Unit = {

diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
@@ -568,29 +568,29 @@ class MainBuildAuxiliaryDockerRequirementsApkTest extends AbstractMainBuildAuxil
   test("test_setup; check the fortune package is added for the test option", DockerTest) { f =>
     val newConfigFilePath = deriveEngineConfig(None, Some("""[{ "type": "apk", "packages": ["fortune"] }]"""), "apk_test_fortune_test")
 
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       newConfigFilePath
     )
 
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("SUCCESS! All 1 out of 1 test scripts succeeded!"))
-    assert(testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("SUCCESS! All 1 out of 1 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
   }
 
   test("test_setup; check the fortune package is not added for the test option when not specified", DockerTest) { f =>
     val newConfigFilePath = deriveEngineConfig(None, None, "apk_base_test")
 
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "-k", "false",
       newConfigFilePath
     )
 
-    assert(testOutput.exceptionText == "Only 0 out of 1 test scripts succeeded!")
+    assert(testOutput.exceptionText.get == "Only 0 out of 1 test scripts succeeded!")
 
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(testOutput.output.contains("ERROR! Only 0 out of 1 test scripts succeeded!"))
-    assert(testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("ERROR! Only 0 out of 1 test scripts succeeded!"))
+    assert(testOutput.stdout.contains("Cleaning up temporary directory"))
   }
 }
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala
@@ -77,14 +77,14 @@ class MainBuildAuxiliaryDockerResourceCopying extends AnyFunSuite with BeforeAnd
   test("Check resources with unsupported format") {
     val configResourcesUnsupportedProtocolFile = configDeriver.derive(""".functionality.resources := [{type: "bash_script", path: "./check_bash_version.sh"}, {path: "ftp://ftp.ubuntu.com/releases/robots.txt"}]""", "config_resource_unsupported_protocol").toString
     // generate viash script
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "build",
       "--engine", "docker",
       "-o", tempFolStr,
       configResourcesUnsupportedProtocolFile
     )
 
-    assert(testOutput.exceptionText == "Unsupported scheme: ftp")
+    assert(testOutput.exceptionText.get == "Unsupported scheme: ftp")
   }
 
   override def afterAll(): Unit = {

diff --git a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
@@ -29,21 +29,21 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
     Files.copy(tmpFolderResourceSourceFile, tmpFolderResourceDestinationFile, StandardCopyOption.REPLACE_EXISTING)
 
     // generate viash script
-    val testText = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "test",
       "--engine", "docker",
       "-k", "true",
       configFile
     )
 
     // basic checks to see if standard test/build was correct
-    assert(testText.contains("Running tests in temporary directory: "))
-    assert(testText.contains("WARNING! No tests found!"))
-    assert(!testText.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(testOutput.stdout.contains("WARNING! No tests found!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
     val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r
 
-    val tempPath = testText.replaceAll("\n", "") match {
+    val tempPath = testOutput.stdout.replaceAll("\n", "") match {
       case FolderRegex(path) => path
       case _ => ""
     }
@@ -77,27 +77,27 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
     }
 
     Directory(tmpFolderResourceDestinationFolder).deleteRecursively()
-    checkTempDirAndRemove(testText, true, "viash_test_auxiliary_resources")
+    checkTempDirAndRemove(testOutput.stdout, true, "viash_test_auxiliary_resources")
   }
 
   test("Check resources with unsupported format", DockerTest) {
     val configResourcesUnsupportedProtocolFile = configDeriver.derive(""".functionality.resources := [{type: "bash_script", path: "./check_bash_version.sh"}, {path: "ftp://ftp.ubuntu.com/releases/robots.txt"}]""", "config_resource_unsupported_protocol").toString
     // generate viash script
-    val testOutput = TestHelper.testMainException2[RuntimeException](
+    val testOutput = TestHelper.testMainException[RuntimeException](
       "test",
       "--engine", "docker",
       "-k", "true",
       configResourcesUnsupportedProtocolFile
     )
 
-    assert(testOutput.exceptionText == "Unsupported scheme: ftp")
+    assert(testOutput.exceptionText.get == "Unsupported scheme: ftp")
 
     // basic checks to see if standard test/build was correct
-    assert(testOutput.output.contains("Running tests in temporary directory: "))
-    assert(!testOutput.output.contains("WARNING! No tests found!"))
-    assert(!testOutput.output.contains("Cleaning up temporary directory"))
+    assert(testOutput.stdout.contains("Running tests in temporary directory: "))
+    assert(!testOutput.stdout.contains("WARNING! No tests found!"))
+    assert(!testOutput.stdout.contains("Cleaning up temporary directory"))
 
-    checkTempDirAndRemove(testOutput.output, true, "viash_test_auxiliary_resources")
+    checkTempDirAndRemove(testOutput.stdout, true, "viash_test_auxiliary_resources")
   }
 
   /**

diff --git a/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala b/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala
@@ -36,7 +36,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "commands_default"
     )
 
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -45,7 +45,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), stdout)
+    assert(testOutput.stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), testOutput.stdout)
   }
 
   test("Verify adding extra commands to verify", DockerTest) {
@@ -54,7 +54,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "commands_extra"
     )
 
-    val (stdout, _, _) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -63,7 +63,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), stdout)
+    assert(testOutput.stdout.matches("\\[notice\\] Building container 'testbash:0\\.1' with Dockerfile\\s*"), testOutput.stdout)
   }
 
   test("Verify base adding an extra required command that doesn't exist", DockerTest) {
@@ -72,7 +72,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "non_existing_command"
     )
 
-    val stdout = TestHelper.testMain(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -81,14 +81,14 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stdout.contains("[notice] Building container 'testbash:0.1' with Dockerfile"))
-    assert(stdout.contains("[error] Docker container 'testbash:0.1' does not contain command 'non_existing_command'."))
+    assert(testOutput.stdout.contains("[notice] Building container 'testbash:0.1' with Dockerfile"))
+    assert(testOutput.stdout.contains("[error] Docker container 'testbash:0.1' does not contain command 'non_existing_command'."))
   }
 
   test("Check deprecated warning", DockerTest) {
     val newConfigFilePath = configDeriver.derive(""".functionality.status := "deprecated"""", "deprecated")
 
-    val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr(
+    val testOutput = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",
@@ -97,8 +97,8 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "--setup", "alwaysbuild"
     )
 
-    assert(stderr.contains("The status of the component 'testbash' is set to deprecated."))
-    assert(exitCode == 0)
+    assert(testOutput.stderr.contains("The status of the component 'testbash' is set to deprecated."))
+    assert(testOutput.exitCode == Some(0))
   }
 
   test("Check component works when multiple_sep is set to ;", DockerTest) {
@@ -107,7 +107,7 @@ class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll {
       "multiple_sep"
     )
 
-    val _ = TestHelper.testMainWithStdErr(
+    val _ = TestHelper.testMain(
       "build",
       "--engine", "docker",
       "--runner", "docker",