From 0750a33206a70dbf00d2997a20bc1ae23e318e89 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Mon, 16 Oct 2023 16:37:52 +0200 Subject: [PATCH] more vdsl3 testing wip --- .../src/test_wfs/concurrency/config.vsh.yaml | 15 ++ .../src/test_wfs/concurrency/main.nf | 104 ++++++++++++++ .../src/test_wfs/runeach/config.vsh.yaml | 15 ++ .../src/test_wfs/runeach/main.nf | 124 ++++++++++++++++ .../nextflow/NextflowScriptTest.scala | 134 +++++++++++------- 5 files changed, 337 insertions(+), 55 deletions(-) create mode 100644 src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/config.vsh.yaml create mode 100644 src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf create mode 100644 src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/config.vsh.yaml create mode 100644 src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/config.vsh.yaml new file mode 100644 index 000000000..0f41e05ae --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/config.vsh.yaml @@ -0,0 +1,15 @@ +functionality: + name: concurrency + namespace: test_wfs + resources: + - type: nextflow_script + path: main.nf + entrypoint: base + # TODO: make absolute when the ns build uses the right CWD + - path: ../../../resources + dependencies: + - name: step1 + - name: step2 + - name: step3 +platforms: + - type: nextflow \ No newline at end of file diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf b/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf new file mode 100644 index 000000000..d16051213 --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/main.nf @@ -0,0 +1,104 @@ +workflow base { + take: input_ch + main: + + // generate list from 0 to 1000 + ch = Channel.fromList(0..1000) + | map { num -> + // create temporary file + file = tempFile() + file.write("num: $num") + + ["num$num", [ num: num, file: file ]] + } + + // test fromstate and tostate with list[string] + | step1.run( + fromState: ["input": "file"], + toState: ["step1_output": "output"] + ) + + | view{ id, state -> + def num = state.num + + // check id + assert id == "num$num": "id should be 'num$num'. Found: '$id'" + + // check file text + def file_text = state.file.toFile().readLines()[0] + assert file_text == "num: $num": "file text should be 'num: $num'. Found: '$file_text'" + + // check step1_output text + def step1_output_text = state.step1_output.toFile().readLines()[0] + assert step1_output_text == "num: $num": "step1_output text should be 'num: $num'. Found: '$step1_output_text'" + + if (num == 0) { + "after step1: id: $id, state: $state" + } else { + null + } + } + + | step1.run( + key: "step1bis", + // TODO: renameKeys, map, mapId, mapData will be deprecated + // TODO: test filter, runIf + map: { id, state -> + def new_state = state + [ + "oid": id, + "extra_key": "foo" + ] + [id, new_state] + }, + mapId: { id -> + "${id}_modified" + }, + mapData: { state -> + state + [another_key: "bar"] + }, + renameKeys: ["original_id": "oid"], + fromState: { id, state -> + ["input": state.file] + }, + toState: { id, output, state -> + state + ["step1bis_output": output.output] + } + ) + + | view { id, state -> + def num = state.num + + // check id + assert id == "num${num}_modified": "id should be 'num${num}_modified'. Found: '$id'" + + // check orig id + assert state.original_id == "num$num": "original_id should be 'num$num'. Found: '${state.original_id}'" + + // check file text + def file_text = state.file.toFile().readLines()[0] + assert file_text == "num: $num": "file text should be 'num: $num'. Found: '$file_text'" + + // check step1_output text + def step1_output_text = state.step1_output.toFile().readLines()[0] + assert step1_output_text == "num: $num": "step1_output text should be 'num: $num'. Found: '$step1_output_text'" + + // check step1bis_output text + def step1bis_output_text = state.step1bis_output.toFile().readLines()[0] + assert step1bis_output_text == "num: $num": "step1bis_output text should be 'num: $num'. Found: '$step1bis_output_text'" + + // check extra_key + assert state.extra_key == "foo": "extra_key should be 'foo'. Found: '${state.extra_key}'" + + // check another_key + assert state.another_key == "bar": "another_key should be 'bar'. Found: '${state.another_key}'" + + if (num == 0) { + "after step1bis: id: $id, state: $state" + } else { + null + } + } + + emit: + input_ch +} diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/config.vsh.yaml new file mode 100644 index 000000000..4328e1860 --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/config.vsh.yaml @@ -0,0 +1,15 @@ +functionality: + name: runeach + namespace: test_wfs + resources: + - type: nextflow_script + path: main.nf + entrypoint: base + # TODO: make absolute when the ns build uses the right CWD + - path: ../../../resources + dependencies: + - name: step1 + - name: step2 + - name: step3 +platforms: + - type: nextflow \ No newline at end of file diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf b/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf new file mode 100644 index 000000000..0bbba13f9 --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/runeach/main.nf @@ -0,0 +1,124 @@ +workflow base { + take: input_ch + main: + + comps = [ + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" + ] + .collect{ let -> + step1.run(key: "step1" + let) + } + + // generate list from 0 to 1000 + ch = Channel.fromList(0..100) + + | map { num -> + // create temporary file + file = tempFile() + file.write("num: $num") + + ["num$num", [ num: num, file: file ]] + } + + | step1.run( + fromState: ["input": "file"], + toState: ["step1_output": "output"] + ) + + + | step1.run( + key: "step1bis", + fromState: { id, state -> + ["input": state.step1_output] + }, + toState: { id, output, state -> + state + ["step1bis_output": output.output] + } + ) + + | runEach( + components: comps, + filter: { id, state, comp -> + id != "num0" || comp.name != "step1a" + }, + id: { id, state, comp -> + "${id}_${comp.name}".toString() + }, + fromState: { id, state, comp -> + ["input": state.step1bis_output] + }, + toState: { id, output, state, comp -> + state + [ + "runEach_output": output.output, + "runEach_key": comp.name, + "extra_key": "foo" + ] + } + // todo: test filter + // todo: test fromState and toState as maps or arrays + ) + + | view { id, state -> + def num = state.num + + // check id + assert id ==~ /num${num}_step1[a-z]/: "id should match 'num${num}_step1[a-z]'. Found: '$id'" + + // check file text + def file_text = state.file.toFile().readLines()[0] + assert file_text == "num: $num": "file text should be 'num: $num'. Found: '$file_text'" + + // check runEach_output text + def runEach_output_text = state.runEach_output.toFile().readLines()[0] + assert runEach_output_text == "num: $num": "runEach_output text should be 'num: $num'. Found: '$runEach_output_text'" + + // check extra_key + assert state.extra_key == "foo": "extra_key should be 'foo'. Found: '${state.extra_key}'" + + // check runEach_key + assert state.runEach_key ==~ /step1[a-z]/: "runEach_key should match 'step1[a-z]'. Found: '${state.runEach_key}'" + + if (num == 0) { + "after runEach: id: $id, state: $state" + } else { + null + } + } + + | toSortedList{ a, b -> a[0] <=> b[0] } + | view { list -> + assert list.size() == 100 * 26 + 1: "list size should be 100 * 25 + 1. Found: ${list.size()}" + + def ids = list.collect{it[0]} + def expectedIds = (0..100).collectMany{ num -> + if (num == 0) { + ["num0_step1a"] + } else { + comps.collect{ comp -> + "num${num}_${comp.name}".toString() + } + } + } + def unexpectedIds = ids - expectedIds + def missingIds = expectedIds - ids + + // println() + // println() + // println("ids: $ids") + // println() + // println("expectedIds: $expectedIds") + // println() + // println("unexpectedIds: $unexpectedIds") + // println() + // println("missingIds: $missingIds") + // println() + // println() + assert unexpectedIds.size() == 0: "unexpected ids: $unexpectedIds" + assert missingIds.size() == 0: "missing ids: $missingIds" + + null + } + emit: + input_ch +} diff --git a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala index ccedae480..b8c89b559 100644 --- a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala @@ -46,51 +46,75 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { ) } - test("Run config pipeline", NextflowTest) { - val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( - mainScript = "target/nextflow/wf/main.nf", - args = List( - "--id", "foo", - "--input1", "resources/lines*.txt", - "--input2", "resources/lines3.txt", - "--publish_dir", "output" - ), - cwd = tempFolFile - ) - - assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - } - - // TODO: use TestHelper.testMainWithStdErr instead of NextflowTestHelper.run; i.e. viash test - test("Test workflow", DockerTest, NextflowTest) { - val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( - mainScript = "target/nextflow/wf/main.nf", - entry = Some("test_base"), - args = List( - "--rootDir", tempFolStr, - "--publish_dir", "output" - ), - cwd = tempFolFile - ) - - assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - } + // test("Run config pipeline", NextflowTest) { + // val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + // mainScript = "target/nextflow/wf/main.nf", + // args = List( + // "--id", "foo", + // "--input1", "resources/lines*.txt", + // "--input2", "resources/lines3.txt", + // "--publish_dir", "output" + // ), + // cwd = tempFolFile + // ) + + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + // } + + // // TODO: use TestHelper.testMainWithStdErr instead of NextflowTestHelper.run; i.e. viash test + // test("Test workflow", DockerTest, NextflowTest) { + // val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + // mainScript = "target/nextflow/wf/main.nf", + // entry = Some("test_base"), + // args = List( + // "--rootDir", tempFolStr, + // "--publish_dir", "output" + // ), + // cwd = tempFolFile + // ) + + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + // } - test("Test fromState/toState", DockerTest, NextflowTest) { - val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( - mainScript = "target/nextflow/test_wfs/fromstate_tostate/main.nf", - args = List( - "--publish_dir", "output" - ), - cwd = tempFolFile - ) - - assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - } - - test("Test filter/runIf", DockerTest, NextflowTest) { + // test("Test fromState/toState", DockerTest, NextflowTest) { + // val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + // mainScript = "target/nextflow/test_wfs/fromstate_tostate/main.nf", + // args = List( + // "--publish_dir", "output" + // ), + // cwd = tempFolFile + // ) + + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + // } + + // test("Test filter/runIf", DockerTest, NextflowTest) { + // val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + // mainScript = "target/nextflow/test_wfs/filter_runif/main.nf", + // args = List( + // "--publish_dir", "output" + // ), + // cwd = tempFolFile + // ) + + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + // } + + // test("Test for concurrency issues", DockerTest, NextflowTest) { + // val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + // mainScript = "target/nextflow/test_wfs/concurrency/main.nf", + // args = List( + // "--publish_dir", "output" + // ), + // cwd = tempFolFile + // ) + + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + // } + + test("Test runEach", DockerTest, NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( - mainScript = "target/nextflow/test_wfs/filter_runif/main.nf", + mainScript = "target/nextflow/test_wfs/runeach/main.nf", args = List( "--publish_dir", "output" ), @@ -101,20 +125,20 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { } - test("Check whether --help is same as Viash's --help", NextflowTest) { - // except that WorkflowHelper.nf will not print alternatives, and - // will always prefix argument names with -- (so --foo, not -f or foo). + // test("Check whether --help is same as Viash's --help", NextflowTest) { + // // except that WorkflowHelper.nf will not print alternatives, and + // // will always prefix argument names with -- (so --foo, not -f or foo). - // run WorkflowHelper's --help - val (exitCode, stdOut1, stdErr1) = NextflowTestHelper.run( - mainScript = "target/nextflow/wf/main.nf", - args = List("--help"), - quiet = true, - cwd = tempFolFile - ) + // // run WorkflowHelper's --help + // val (exitCode, stdOut1, stdErr1) = NextflowTestHelper.run( + // mainScript = "target/nextflow/wf/main.nf", + // args = List("--help"), + // quiet = true, + // cwd = tempFolFile + // ) - assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut1\nStd error:\n$stdErr1") - } + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut1\nStd error:\n$stdErr1") + // } override def afterAll(): Unit = {