Merge pull request #577 from viash-io/develop_0_8

Release Viash 0.8.0

CHANGELOG.md

@@ -1,64 +1,7 @@
-# Viash 0.8.0-RC6 (2023-10-11): fix race condition issue in runEach
-
-This part of the changelog will be removed.
-
-# Viash 0.8.0-RC5 (2023-10-11): Fix run workflow
-
-This part of the changelog will be removed.
-
-* fix run workflow
-
-* fix 'safe'join helper function
-
-* fix 'safe'join helper once more
-
-* fix wf factory
-
-# Viash 0.8.0-RC4 (2023-10-10): Refactor Nextflow helper functions
-
-This part of the changelog will be removed.
-
-## MAJOR CHANGES
-
-* `NextflowPlatform`: Refactoring of helper functions (PR #557).
-  - Cleaned up `processConfig()`: Removed support for `functionality.inputs` and `functionality.outputs`
-  - Cleaned up `processConfig()`: Removed support for `.functionality.argument_groups[].argument` containing a list of argument ids as opposed to the arguments themselves.
-  - Rewrote `--param_list` parser.
-  - Removed unused function `applyConfig()` and `applyConfigToOneParamSet()`.
-  - Refactored `channelFromParams()` to make use of new helper functions.
-  - Removed deprecated `paramsToChannel()`, `paramsToList()`, `viashChannel()`.
-  - Deprecated `preprocessInputs()` -- use the wrapped Viash Nextflow functionality instead.
-  - Refactored `preprocessInputs()` to make use of new helper functions.
-  - Reprecated run arguments `map`, `mapData`, `mapPassthrough`, `renameKeys`.
-
-
-# Viash 0.8.0-RC3 (2023-10-07): More bugfixes, more Nextflow functionality
-
-This release contains more bugfixes related to the Nextflow code generation functionality.
-It also adds a new `runIf` argument to the `NextflowPlatform` which allows for conditional execution of modules.
-
-## NEW FUNCTIONALITY
-
-* `NextflowPlatform`: Added new `.run()` argument `runIf` - a function that determines whether the module should be run or not (PR #553).
-  If the `runIf` closure evaluates to `true`, then the module will be run. Otherwise it will be passed through without running.
-
-## MINOR CHANGES
-
-* `NextflowPlatform`: Rename internal VDSL3 variables to be more consistent with regular Viash component variables and avoid naming clashes (PR #553).
-
-## BUG FIXES
-
-* `export cli_autocomplete`: Fix output script format and hide `--loglevel` and `--colorize` (PR #544). Masked arguments are usable but might not be very useful to always display in help messages.
-
-# Viash 0.8.0-RC2 (2023-10-04): Some bugfixes
-
-Some bugfixes related to the new dependencies and Nextflow code generation functionality.
-
-This part of the changelog will be removed.
-
-# Viash 0.8.0-RC1 (2023-10-02): Nextflow workflows and dependencies
+# Viash 0.8.0 (2023-10-23): Nextflow workflows and dependencies
 
 Nextflow workflows definitions are picked up by Viash and assembled into a functional Nextflow workflow, reducing the amount of boilerplate code needed to be written by the user.
+It also adds a new `runIf` argument to the `NextflowPlatform` which allows for conditional execution of modules.
 We added new 'dependencies' functionality to allow for more advanced functionality to be offloaded and re-used in components and workflows.
 
 ## BREAKING CHANGES
@@ -102,6 +45,8 @@ We added new 'dependencies' functionality to allow for more advanced functionali
   Example input event: `["id", [input: file(...)]]`,
   Example output event: `["newid", [output: file(...), _meta: [join_id: "id"]]]`
 
+* `NextflowPlatform`: Added new `.run()` argument `runIf` - a function that determines whether the module should be run or not (PR #553).
+  If the `runIf` closure evaluates to `true`, then the module will be run. Otherwise it will be passed through without running.
 
 ## MAJOR CHANGES
 
@@ -111,12 +56,25 @@ We added new 'dependencies' functionality to allow for more advanced functionali
 
 * `NextflowPlatform`: Set default tag to `"$id"` (#521, PR #518).
 
+* `NextflowPlatform`: Refactoring of helper functions (PR #557).
+  - Cleaned up `processConfig()`: Removed support for `functionality.inputs` and `functionality.outputs`
+  - Cleaned up `processConfig()`: Removed support for `.functionality.argument_groups[].argument` containing a list of argument ids as opposed to the arguments themselves.
+  - Rewrote `--param_list` parser.
+  - Removed unused function `applyConfig()` and `applyConfigToOneParamSet()`.
+  - Refactored `channelFromParams()` to make use of new helper functions.
+  - Removed deprecated `paramsToChannel()`, `paramsToList()`, `viashChannel()`.
+  - Deprecated `preprocessInputs()` -- use the wrapped Viash Nextflow functionality instead.
+  - Refactored `preprocessInputs()` to make use of new helper functions.
+  - Reprecated run arguments `map`, `mapData`, `mapPassthrough`, `renameKeys`.
+
 ## MINOR CHANGES
 
 * `NextflowPlatform`: Throw error when unexpected keys are passed to the `.run()` method (#512, PR #518).
 
 * `NextflowPlatform`: Use `moduleDir` instead of `projectDir` to determine the resource directory.
 
+* `NextflowPlatform`: Rename internal VDSL3 variables to be more consistent with regular Viash component variables and avoid naming clashes (PR #553).
+
 ## DOCUMENTATION
 
 * Minor fixes to VDSL3 reference documentation (PR #508).
@@ -127,6 +85,8 @@ We added new 'dependencies' functionality to allow for more advanced functionali
 
 * `NextflowPlatform`: When using `fromState` and `toState`, do not throw an error when the state or output is missing an optional argument (PR #515).
 
+* `export cli_autocomplete`: Fix output script format and hide `--loglevel` and `--colorize` (PR #544). Masked arguments are usable but might not be very useful to always display in help messages.
+
 # Viash 0.7.5 (2023-08-11): Minor breaking changes and new features
 
 This release contains minor breaking change due to deprecated or outdated functionality being removed.

build.sbt

@@ -1,6 +1,6 @@
 name := "viash"
 
-version := "0.8.0-RC6"
+version := "0.8.0"
 
 scalaVersion := "2.13.10"
 

src/main/resources/io/viash/platforms/nextflow/channel/channelFromParams.nf

@@ -113,6 +113,6 @@ private List<Tuple2<String, Map<String, Object>>> _paramsToParamSets(Map params,
  *
  */
 def channelFromParams(Map params, Map config) {
-  processedParams = _paramsToParamSets(params, config)
+  def processedParams = _paramsToParamSets(params, config)
   return Channel.fromList(processedParams)
 }

src/main/resources/io/viash/platforms/nextflow/channel/preprocessInputs.nf

@@ -34,7 +34,7 @@ def preprocessInputsDeprecationWarning() {
 def preprocessInputs(Map args) {
   preprocessInputsDeprecationWarning()
 
-  config = args.config
+  def config = args.config
   assert config instanceof Map : 
     "Error in preprocessInputs: config must be a map. " +
     "Expected class: Map. Found: config.getClass() is ${config.getClass()}"

src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf

@@ -166,7 +166,7 @@ def publishStatesByConfig(Map args) {
                   filenameTemplate = filenameTemplate[0]
                 }
                 // instantiate the template
-                filename = filenameTemplate
+                def filename = filenameTemplate
                   .replaceAll('\\$id', id_)
                   .replaceAll('\\$key', key_)
                 if (par.multiple) {

src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf

@@ -99,12 +99,9 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
         tuple
       }
 
-    if (workflowArgs.filter) {
-      chModifiedFiltered = chModified
-        | filter{workflowArgs.filter(it)}
-    } else {
-      chModifiedFiltered = chModified
-    }
+    chModifiedFiltered = workflowArgs.filter ?
+      chModified | filter{workflowArgs.filter(it)} :
+      chModified
 
     if (workflowArgs.runIf) {
       runIfBranch = chModifiedFiltered.branch{ tup ->
@@ -118,15 +115,12 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
       chPassthrough = Channel.empty()
     }
 
-    if (workflowArgs.fromState) {
-      chArgs = chRun
-        | map{
-          def new_data = workflowArgs.fromState(it.take(2))
-          [it[0], new_data]
-        }
-    } else {
-      chArgs = chRun
-    }
+    chArgs = workflowArgs.fromState ? 
+      chRun | map{
+        def new_data = workflowArgs.fromState(it.take(2))
+        [it[0], new_data]
+      } :
+      chRun
 
     // fill in defaults
     chArgsWithDefaults = chArgs

src/main/scala/io/viash/functionality/dependencies/Dependency.scala

@@ -30,10 +30,10 @@ import io.viash.exceptions.MissingBuildYamlException
     |""".stripMargin)
 @exampleWithDescription(
   """name: qc/multiqc
-    |  repository: 
-    |    type: github
-    |    uri: openpipelines-bio/modules
-    |    tag: 0.3.0
+    |repository: 
+    |  type: github
+    |  repo: openpipelines-bio/modules
+    |  tag: 0.3.0
     |""".stripMargin,
   "yaml",
   "Definition of dependency with a fully defined repository"
@@ -47,11 +47,17 @@ import io.viash.exceptions.MissingBuildYamlException
 )
 @exampleWithDescription(
   """name: qc/multiqc
-    |  repository: "openpipelines-bio"
+    |repository: "openpipelines-bio"
     |""".stripMargin,
   "yaml",
   "Definition of a dependency with a repository defined as 'openpipelines-bio' under `.functionality.repositories`."
 )
+@exampleWithDescription(
+  """name: qc/multiqc
+    |""".stripMargin,
+  "yaml",
+  "Definition of a local dependency. This dependency is present in the current code base and will be built when `viash ns build` is run."
+)
 case class Dependency(
   @description("The full name of the dependency component. This should include the namespace.")
   @example("name: \"my_namespace\"component", "yaml")

src/main/scala/io/viash/functionality/dependencies/LocalRepository.scala

@@ -26,6 +26,14 @@ import java.nio.file.Paths
     |Alternatively, this can be used to refer to a code repository present on the local hard-drive instead of fetchable remotely, for example during development.
     |""".stripMargin
 )
+@exampleWithDescription(
+  """name: my_local_code
+    |type: local
+    |path: /additional_code/src
+    |""".stripMargin,
+  "yaml",
+  "Refer to a local code repository under `additional_code/src` referenced to the Viash Project Config file."
+)
 @subclass("local")
 case class LocalRepository(
   name: String = "",

src/main/scala/io/viash/platforms/NextflowPlatform.scala

@@ -180,8 +180,10 @@ case class NextflowPlatform(
     val processLabels = config.labels.map{ case (k, v) => s"withLabel: $k { $v }"}
     val inlineScript = config.script.toList
 
+    val name = f"${functionality.namespace.map(_ + "/").getOrElse("")}${functionality.name}"
+
     s"""manifest {
-    |  name = '${functionality.name}'
+    |  name = '${name}'
     |  mainScript = 'main.nf'
     |  nextflowVersion = '!>=20.12.1-edge'$versStr$descStr$authStr
     |}$profileStr
@@ -196,7 +198,6 @@ case class NextflowPlatform(
 
   // interpreted from BashWrapper
   def renderMainNf(config: Config, containerDirective: Option[DockerImageInfo]): String = {
-
     if (config.functionality.mainScript.isEmpty) {
       throw new RuntimeException("No main script defined")
     }

src/test/resources/testnextflowvdsl3/src/sub_workflow/config.vsh.yaml

@@ -0,0 +1,17 @@
+functionality:
+  name: sub_workflow
+  arguments:
+    - name: "--file"
+      type: file
+      direction: input
+    - name: "--output"
+      type: file
+      direction: output
+  resources:
+    - type: nextflow_script
+      path: main.nf
+      entrypoint: base
+  dependencies:
+    - name: step1
+platforms:
+  - type: nextflow

src/test/resources/testnextflowvdsl3/src/sub_workflow/main.nf

@@ -0,0 +1,16 @@
+workflow base {
+  take: input_ch
+  main:
+
+    output_ch = input_ch
+    | step1.run(
+      fromState: ["input": "file"],
+      toState: { id, output, state -> 
+        def newState = ["output":  output.output]
+        return newState
+      }
+    )
+
+  emit:
+  output_ch
+}

src/test/resources/testnextflowvdsl3/src/test_wfs/concurrency/config.vsh.yaml

@@ -0,0 +1,15 @@
+functionality:
+  name: concurrency
+  namespace: test_wfs
+  resources:
+    - type: nextflow_script
+      path: main.nf
+      entrypoint: base
+    # TODO: make absolute when the ns build uses the right CWD
+    - path: ../../../resources
+  dependencies:
+    - name: step1
+    - name: step2
+    - name: step3
+platforms:
+  - type: nextflow