Merge pull request #542 from viash-io/develop

Viash 0.8.0 RC2

CHANGELOG.md

@@ -1,3 +1,9 @@
+# Viash 0.8.0-RC2 (2023-10-04): Some bugfixes
+
+Some bugfixes related to the new dependencies and Nextflow code generation functionality.
+
+This part of the changelog will be removed.
+
 # Viash 0.8.0-RC1 (2023-10-02): Nextflow workflows and dependencies
 
 Nextflow workflows definitions are picked up by Viash and assembled into a functional Nextflow workflow, reducing the amount of boilerplate code needed to be written by the user.

build.sbt

@@ -1,6 +1,6 @@
 name := "viash"
 
-version := "0.8.0-RC1"
+version := "0.8.0-RC2"
 
 scalaVersion := "2.13.10"
 
@@ -44,6 +44,7 @@ generateWorkflowHelper := {
   import sbt._
   import java.nio.file._
 
+  val rootDir = (Compile / baseDirectory).value
   val basePath = (Compile / resourceDirectory).value / "io" / "viash" / "platforms" / "nextflow"
   val wfHelper = Paths.get(basePath.toString, "WorkflowHelper.nf")
 
@@ -64,7 +65,8 @@ generateWorkflowHelper := {
     StandardOpenOption.CREATE)
 
   files.sorted.foreach { path =>
-    Files.write(wfHelper, s"\n// helper file: '${path}'\n".getBytes(), StandardOpenOption.APPEND)
+    val relativePath = rootDir.relativize(path)
+    Files.write(wfHelper, s"\n// helper file: '${relativePath.get}'\n".getBytes(), StandardOpenOption.APPEND)
     Files.write(wfHelper, Files.readAllBytes(path.toPath()), StandardOpenOption.APPEND)
   }
 

src/main/resources/io/viash/platforms/nextflow/arguments/processInputsOutputs.nf

@@ -28,6 +28,14 @@ def typeCheck(String stage, Map par, Object value, String id, String key) {
       value = value.toLong()
     }
     expectedClass = value instanceof Long ? null : "Long"
+  } else if (par.type == "double") {
+    if (value instanceof java.math.BigDecimal) {
+      value = value.doubleValue()
+    }
+    if (value instanceof Float || value instanceof Integer || value instanceof Long) {
+      value = value.toDouble()
+    }
+    expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
     expectedClass = value instanceof Boolean ? null : "Boolean"
   } else if (par.type == "file") {

src/main/resources/io/viash/platforms/nextflow/readwrite/toJsonBlob.nf

@@ -1,3 +1,3 @@
-String toJsonBlob(Map data) {
+String toJsonBlob(data) {
   return groovy.json.JsonOutput.toJson(data)
 }

src/main/resources/io/viash/platforms/nextflow/readwrite/toTaggedYamlBlob.nf

@@ -27,7 +27,7 @@ class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer {
   }
 }
 
-String toTaggedYamlBlob(Map data) {
+String toTaggedYamlBlob(data) {
   def options = new org.yaml.snakeyaml.DumperOptions()
   options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK)
   def representer = new CustomRepresenter(options)

src/main/resources/io/viash/platforms/nextflow/readwrite/toYamlBlob.nf

@@ -1,4 +1,4 @@
-String toYamlBlob(Map data) {
+String toYamlBlob(data) {
   def options = new org.yaml.snakeyaml.DumperOptions()
   options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK)
   options.setPrettyFlow(true)

src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf

@@ -40,6 +40,8 @@ def collectInputOutputPaths(obj, prefix) {
 
 def publishStates(Map args) {
   def key_ = args.get("key")
+
+  assert key_ != null : "publishStates: key must be specified"
 
   workflow publishStatesWf {
     take: input_ch
@@ -105,8 +107,11 @@ process publishStatesProc {
 
 // this assumes that the state contains no other values other than those specified in the config
 def publishStatesByConfig(Map args) {
-  def key_ = args.get("key")
   def config = args.get("config")
+  assert config != null : "publishStatesByConfig: config must be specified"
+
+  def key_ = args.get("key", config.functionality.name)
+  assert key_ != null : "publishStatesByConfig: key must be specified"
 
   workflow publishStatesSimpleWf {
     take: input_ch

src/main/scala/io/viash/Main.scala

@@ -254,8 +254,8 @@ object Main extends Logging {
         )
         0 // Exceptions are thrown when a test fails, so then the '0' is not returned but a '1'. Can be improved further.
       case List(cli.namespace, cli.namespace.build) =>
-        val configs = readConfigs(cli.namespace.build, project = proj1)
-        val configs2 = namespaceDependencies(configs, proj1.target, proj1.rootDir)
+        val (configs, allConfigs) = readConfigs(cli.namespace.build, project = proj1)
+        val configs2 = namespaceDependencies(configs, allConfigs, proj1.target, proj1.rootDir)
         var buildResults = ViashNamespace.build(
           configs = configs2,
           target = proj1.target.get,
@@ -269,8 +269,8 @@ object Main extends Logging {
           .count(_.isError)
         if (errors > 0) 1 else 0
       case List(cli.namespace, cli.namespace.test) =>
-        val configs = readConfigs(cli.namespace.test, project = proj1)
-        val configs2 = namespaceDependencies(configs, None, proj1.rootDir)
+        val (configs, allConfigs) = readConfigs(cli.namespace.test, project = proj1)
+        val configs2 = namespaceDependencies(configs, allConfigs, None, proj1.rootDir)
         val testResults = ViashNamespace.test(
           configs = configs2,
           parallel = cli.namespace.test.parallel(),
@@ -284,13 +284,13 @@ object Main extends Logging {
         val errors = testResults.flatMap(_.toOption).count(_.isError)
         if (errors > 0) 1 else 0
       case List(cli.namespace, cli.namespace.list) =>
-        val configs = readConfigs(
+        val (configs, allConfigs) = readConfigs(
           cli.namespace.list,
           project = proj1,
           addOptMainScript = false, 
           applyPlatform = cli.namespace.list.platform.isDefined
         )
-        val configs2 = namespaceDependencies(configs, None, proj1.rootDir)
+        val configs2 = namespaceDependencies(configs, allConfigs, None, proj1.rootDir)
         ViashNamespace.list(
           configs = configs2,
           format = cli.namespace.list.format(),
@@ -299,7 +299,7 @@ object Main extends Logging {
         val errors = configs.flatMap(_.toOption).count(_.isError)
         if (errors > 0) 1 else 0
       case List(cli.namespace, cli.namespace.exec) =>
-        val configs = readConfigs(
+        val (configs, allConfigs) = readConfigs(
           cli.namespace.exec, 
           project = proj1, 
           applyPlatform = cli.namespace.exec.applyPlatform()
@@ -434,15 +434,15 @@ object Main extends Logging {
     project: ViashProject,
     addOptMainScript: Boolean = true,
     applyPlatform: Boolean = true
-  ): List[Either[(Config, Option[Platform]), Status]] = {
+  ): (List[Either[(Config, Option[Platform]), Status]], List[Config]) = {
     val source = project.source.get
     val query = subcommand.query.toOption
     val queryNamespace = subcommand.query_namespace.toOption
     val queryName = subcommand.query_name.toOption
     val platformStr = subcommand.platform.toOption
     val configMods = project.config_mods
 
-    val configs = Config.readConfigs(
+    val (configs, allConfigs) = Config.readConfigs(
       source = source,
       projectDir = project.rootDir.map(_.toUri()),
       query = query,
@@ -452,7 +452,8 @@ object Main extends Logging {
       addOptMainScript = addOptMainScript
     )
 
-    if (applyPlatform) {
+    val appliedPlatformConfigs = 
+      if (applyPlatform) {
       // create regex for filtering platform ids
       val platformStrVal = platformStr.getOrElse(".*")
 
@@ -490,6 +491,8 @@ object Main extends Logging {
         case Left(conf) => Left((conf, None: Option[Platform]))
       }}
     }
+
+    (appliedPlatformConfigs, allConfigs)
   }
 
   // Handle dependencies operations for a single config
@@ -501,15 +504,14 @@ object Main extends Logging {
   }
 
   // Handle dependency operations for namespaces
-  def namespaceDependencies(configs: List[Either[(Config, Option[Platform]), Status]], target: Option[String], rootDir: Option[Path]): List[Either[(Config, Option[Platform]), Status]] = {
+  def namespaceDependencies(configs: List[Either[(Config, Option[Platform]), Status]], allConfigs: List[Config], target: Option[String], rootDir: Option[Path]): List[Either[(Config, Option[Platform]), Status]] = {
     if (target.isDefined)
       DependencyResolver.createBuildYaml(target.get)
 
     configs.map{
       case Left((config: Config, platform: Option[Platform])) => {
         Try{
-          val validConfigs = configs.flatMap(_.swap.toOption).map(_._1)
-          handleSingleConfigDependency(config, platform, target, rootDir, validConfigs)
+          handleSingleConfigDependency(config, platform, target, rootDir, allConfigs)
         }.fold(
           e => e match {
             case de: AbstractDependencyException =>

src/main/scala/io/viash/config/Config.scala

@@ -294,7 +294,7 @@ object Config extends Logging {
     queryName: Option[String] = None,
     configMods: List[String] = Nil,
     addOptMainScript: Boolean = true
-  ): List[Either[Config, Status]] = {
+  ): (List[Either[Config, Status]], List[Config]) = {
 
     val sourceDir = Paths.get(source)
 
@@ -305,7 +305,7 @@ object Config extends Logging {
         attrs.isRegularFile
     })
 
-    scriptFiles.map { file =>
+    val allConfigsWithStdOutErr = scriptFiles.map { file =>
       try {
         // read config to get an idea of the name and namespaces
         // warnings will be captured for now, and will be displayed when reading the second time
@@ -322,7 +322,16 @@ object Config extends Logging {
               )
             }
           }
+          Left((config, stdout, stderr))
+      } catch {
+        case _: Exception =>
+          error(s"Reading file '$file' failed")
+          Right(ParseError)
+      }
+    }
 
+    val filteredConfigs: List[Either[Config, Status]] = allConfigsWithStdOutErr.map { 
+      case Left((config, stdout, stderr)) => 
         val funName = config.functionality.name
         val funNs = config.functionality.namespace
 
@@ -351,11 +360,11 @@ object Config extends Logging {
         } else {
           Right(Disabled)
         }
-      } catch {
-        case _: Exception =>
-          error(s"Reading file '$file' failed")
-          Right(ParseError)
-      }
+      case Right(status) => Right(status)
     }
+
+    val allConfigs = allConfigsWithStdOutErr.collect({ case Left((config, _, _)) if config.functionality.isEnabled => config })
+
+    (filteredConfigs, allConfigs)
   }
 }