Develop

.github/workflows/ns_test.yml

@@ -9,12 +9,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up sbt
+    - name: Set up java
       uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: '11'
 
+    - name: Set up sbt
+      uses: sbt/setup-sbt@v1
+
     - name: Build viash
       run: |
         echo "${HOME}/.local/bin" >> $GITHUB_PATH

.github/workflows/sbt_test.yml

@@ -21,15 +21,9 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: viash-io/viash-actions/update-docker-engine@v5
+    - uses: viash-io/viash-actions/project/update-docker-engine@v6
       if: runner.os == 'Linux'
 
-    - name: Set up Nextflow
-      if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }}
-      uses: nf-core/setup-nextflow@v1
-      with:
-        version: ${{ matrix.java.nxf_ver }}
-
     - name: Set up R
       uses: r-lib/actions/setup-r@v2
       with:
@@ -42,18 +36,14 @@ jobs:
           processx
           testthat
 
-    - name: Set up java & sbt
+    - name: Set up java
       uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: ${{ matrix.java.ver }}
 
-    - name: Set up sbt specifically on macOS on arm64 if needed
-      if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64'}}
-      run: |
-        if ! command -v sbt &> /dev/null; then
-          brew install sbt
-        fi
+    - name: Set up sbt
+      uses: sbt/setup-sbt@v1
 
     - name: Set up Scala
       run: |
@@ -69,6 +59,12 @@ jobs:
       with:
         python-version: '3.x'
 
+    - name: Set up Nextflow
+      if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }}
+      uses: nf-core/setup-nextflow@v2
+      with:
+        version: ${{ matrix.java.nxf_ver }}
+
     - name: Run tests
       run: |
         if [[ "${{ matrix.config.name }}" =~ ^ubuntu.*$ ]] && [[ "${{ matrix.java.run_coverage }}" == "true" ]]; then

CHANGELOG.md

@@ -1,3 +1,37 @@
+# Viash 0.9.1 (2024-12-16): Enhanced nextflow support and Scala 3 update
+
+Workflows can now publish results asynchronously by emitting multiple output channels. These results will then be merged into a published output behind the screens.
+Dependencies will use the new dedicated git url instead of the top level domain name.
+
+## NEW FEATURES
+
+* `Nextflow` runner: allow emitting multiple output channels (PR #736).
+
+* `Scope`: Add a `scope` field to the config (PR #782). This allows tuning how the components is built for release.
+
+## MINOR CHANGES
+
+* `viash-hub`: Change the url for viash-hub Git access to packages.viash-hub.com (PR #774).
+
+* `RRequirements`: Allow single quotes to be used again in the `.script` field (PR #771).
+
+* `scala`: Update Scala to Scala 3 (PR #759).
+  For most of the code, this was a minor update, so no breaking changes are expected.
+  The biggest change is how the exporting of the schema is done, but this has no impact on the user.
+  However, switching to Scala 3 allows for additional features and improvements in the future.
+
+* `--help`: Component `--help` messages will now display what built in `---` options are available (PR #784).
+
+## BUG FIXES
+
+* `config build`: Fix a bug where a missing main script would cause a stack trace instead of a proper error message (PR #776).
+  The error message showed the path of the missing resource but it was easy to miss given the stack trace, besides it shouldn't have been a stack trace anyway.
+
+* `RRequirements`: Treat warnings as errors when installing R dependencies in Docker engines (PR #771).
+
+* `Nextflow` runner: fix false-positive error when output argument arguments `required: true` 
+  are incorrectly flagged as missing input arguments (PR #778).
+
 # Viash 0.9.0 (2024-09-03): Restructure platforms into runners and engines
 
 This release restructures the introduces changes to the Viash config:

build.sbt

@@ -1,32 +1,37 @@
 name := "viash"
 
-version := "0.9.0"
+version := "0.9.1"
 
-scalaVersion := "2.13.14"
+scalaVersion := "3.3.4"
 
 libraryDependencies ++= Seq(
   "org.scalactic" %% "scalactic" % "3.2.15" % "test",
   "org.scalatest" %% "scalatest" % "3.2.15" % "test",
   "org.rogach" %% "scallop" % "5.0.0",
-  "org.scala-lang" % "scala-reflect" % scalaVersion.value,
   "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1",
   "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",
-  "com.github.julien-truffaut" %% "monocle-core"  % "2.1.0",
-  "com.github.julien-truffaut" %% "monocle-macro" % "2.1.0"
+  "dev.optics" %% "monocle-core"  % "3.1.0",
+  "dev.optics" %% "monocle-macro" % "3.1.0"
 )
 
-val circeVersion = "0.14.1"
+val circeVersion = "0.14.7"
 
 libraryDependencies ++= Seq(
   "io.circe" %% "circe-core",
   "io.circe" %% "circe-generic",
   "io.circe" %% "circe-parser",
-  "io.circe" %% "circe-generic-extras",
-  "io.circe" %% "circe-optics",
-  "io.circe" %% "circe-yaml"
+  // "io.circe" %% "circe-generic-extras",
+  // "io.circe" %% "circe-optics",
+  // "io.circe" %% "circe-yaml"
 ).map(_ % circeVersion)
 
-scalacOptions ++= Seq("-unchecked", "-deprecation")
+libraryDependencies ++= Seq(
+  "io.circe" %% "circe-optics" % "0.15.0",
+  "io.circe" %% "circe-yaml" % "0.15.2",
+)
+
+scalacOptions ++= Seq("-unchecked", "-deprecation", "-explain")
+scalacOptions ++= Seq("-Xmax-inlines", "50")
 
 organization := "Data Intuitive"
 startYear := Some(2020)

configure

@@ -12,7 +12,7 @@ while [ $# -gt 0 ]; do
 	        prefix="$2"
 	        shift 2
 	    ;;
-        --help)
+        *)
 	        echo 'usage: ./configure [options]'
 	        echo 'options:'
 	        echo '  --prefix=<path>: installation prefix'

src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf

@@ -1,7 +1,7 @@
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }

src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf

@@ -1,12 +1,5 @@
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
-    config.allArguments.each { arg ->
-      if (arg.direction == "output" && arg.required) {
-        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
-          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
-      }
-    }
-
     outputs = outputs.collectEntries { name, value ->
       def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -18,3 +11,14 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
   }
   return outputs
 }
+
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+  if (!workflow.stubRun) {
+    config.allArguments.each { arg ->
+      if (arg.direction == "output" && arg.required) {
+        assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : 
+          "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+      }
+    }
+  }
+}

src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf

@@ -0,0 +1,154 @@
+def publishFiles(Map args) {
+  def key_ = args.get("key")
+
+  assert key_ != null : "publishFiles: key must be specified"
+
+  workflow publishFilesWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1]
+
+          // the input files and the target output filenames
+          def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+          def inputFiles_ = inputoutputFilenames_[0]
+          def outputFilenames_ = inputoutputFilenames_[1]
+
+          [id_, inputFiles_, outputFilenames_]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesWf
+}
+
+process publishFilesProc {
+  // todo: check publishpath?
+  publishDir path: "${getPublishDir()}/", mode: "copy"
+  tag "$id"
+  input:
+    tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+  output:
+    tuple val(id), path{outputFiles}
+  script:
+  def copyCommands = [
+    inputFiles instanceof List ? inputFiles : [inputFiles],
+    outputFiles instanceof List ? outputFiles : [outputFiles]
+  ]
+    .transpose()
+    .collectMany{infile, outfile ->
+      if (infile.toString() != outfile.toString()) {
+        [
+          "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+          "cp -r '${infile.toString()}' '${outfile.toString()}'"
+        ]
+      } else {
+        // no need to copy if infile is the same as outfile
+        []
+      }
+    }
+  """
+  echo "Copying output files to destination folder"
+  ${copyCommands.join("\n  ")}
+  """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+  def config = args.get("config")
+  assert config != null : "publishFilesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.name)
+  assert key_ != null : "publishFilesByConfig: key must be specified"
+
+  workflow publishFilesSimpleWf {
+    take: input_ch
+    main:
+      input_ch
+        | map { tup ->
+          def id_ = tup[0]
+          def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+          def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+          // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+          //   - key is a String
+          //   - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+          //   - inputPath is a List[Path]
+          //   - outputFilename is a List[String]
+          //   - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+          def processedState =
+            config.allArguments
+              .findAll { it.direction == "output" }
+              .collectMany { par ->
+                def plainName_ = par.plainName
+                // if the state does not contain the key, it's an
+                // optional argument for which the component did 
+                // not generate any output OR multiple channels were emitted
+                // and the output was just not added to using the channel
+                // that is now being parsed
+                if (!state_.containsKey(plainName_)) {
+                  return []
+                }
+                def value = state_[plainName_]
+                // if the parameter is not a file, it should be stored
+                // in the state as-is, but is not something that needs 
+                // to be copied from the source path to the dest path
+                if (par.type != "file") {
+                  return [[inputPath: [], outputFilename: []]]
+                }
+                // if the orig state does not contain this filename,
+                // it's an optional argument for which the user specified
+                // that it should not be returned as a state
+                if (!origState_.containsKey(plainName_)) {
+                  return []
+                }
+                def filenameTemplate = origState_[plainName_]
+                // if the pararameter is multiple: true, fetch the template
+                if (par.multiple && filenameTemplate instanceof List) {
+                  filenameTemplate = filenameTemplate[0]
+                }
+                // instantiate the template
+                def filename = filenameTemplate
+                  .replaceAll('\\$id', id_)
+                  .replaceAll('\\$\\{id\\}', id_)
+                  .replaceAll('\\$key', key_)
+                  .replaceAll('\\$\\{key\\}', key_)
+                if (par.multiple) {
+                  // if the parameter is multiple: true, the filename
+                  // should contain a wildcard '*' that is replaced with
+                  // the index of the file
+                  assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+                  def outputPerFile = value.withIndex().collect{ val, ix ->
+                    def filename_ix = filename.replace("*", ix.toString())
+                    def inputPath = val instanceof File ? val.toPath() : val
+                    [inputPath: inputPath, outputFilename: filename_ix]
+                  }
+                  def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> 
+                    [key, outputPerFile.collect{dic -> dic[key]}]
+                  }
+                  return [[key: plainName_] + transposedOutputs]
+                } else {
+                  def value_ = java.nio.file.Paths.get(filename)
+                  def inputPath = value instanceof File ? value.toPath() : value
+                  return [[inputPath: [inputPath], outputFilename: [filename]]]
+                }
+              }
+
+          def inputPaths = processedState.collectMany{it.inputPath}
+          def outputFilenames = processedState.collectMany{it.outputFilename}
+
+
+          [id_, inputPaths, outputFilenames]
+        }
+        | publishFilesProc
+    emit: input_ch
+  }
+  return publishFilesSimpleWf
+}
+
+
+