Merge remote-tracking branch 'origin/develop' into feature/scala3

.github/workflows/ns_test.yml

@@ -9,12 +9,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up sbt
+    - name: Set up java
       uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: '11'
 
+    - name: Set up sbt
+      uses: sbt/setup-sbt@v1
+
     - name: Build viash
       run: |
         echo "${HOME}/.local/bin" >> $GITHUB_PATH

.github/workflows/sbt_test.yml

@@ -24,12 +24,6 @@ jobs:
     - uses: viash-io/viash-actions/update-docker-engine@v5
       if: runner.os == 'Linux'
 
-    - name: Set up Nextflow
-      if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }}
-      uses: nf-core/setup-nextflow@v1
-      with:
-        version: ${{ matrix.java.nxf_ver }}
-
     - name: Set up R
       uses: r-lib/actions/setup-r@v2
       with:
@@ -42,18 +36,14 @@ jobs:
           processx
           testthat
 
-    - name: Set up java & sbt
+    - name: Set up java
       uses: actions/setup-java@v4
       with:
         distribution: temurin
         java-version: ${{ matrix.java.ver }}
 
-    - name: Set up sbt specifically on macOS on arm64 if needed
-      if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64'}}
-      run: |
-        if ! command -v sbt &> /dev/null; then
-          brew install sbt
-        fi
+    - name: Set up sbt
+      uses: sbt/setup-sbt@v1
 
     - name: Set up Scala
       run: |
@@ -69,6 +59,12 @@ jobs:
       with:
         python-version: '3.x'
 
+    - name: Set up Nextflow
+      if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }}
+      uses: nf-core/setup-nextflow@v1
+      with:
+        version: ${{ matrix.java.nxf_ver }}
+
     - name: Run tests
       run: |
         if [[ "${{ matrix.config.name }}" =~ ^ubuntu.*$ ]] && [[ "${{ matrix.java.run_coverage }}" == "true" ]]; then

build.sbt

@@ -1,6 +1,6 @@
 name := "viash"
 
-version := "0.9.0-dev"
+version := "0.9.1-dev"
 
 scalaVersion := "3.3.4"
 

src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf

@@ -1,7 +1,7 @@
 Map _processInputValues(Map inputs, Map config, String id, String key) {
   if (!workflow.stubRun) {
     config.allArguments.each { arg ->
-      if (arg.required) {
+      if (arg.required && arg.direction == "input") {
         assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : 
           "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
       }

src/main/scala/io/viash/ViashConfig.scala

@@ -49,7 +49,7 @@ object ViashConfig extends Logging{
       throw new ExitException(1)
     }
     // check if we can read code
-    if (config.mainScript.get.read.isEmpty) {
+    if (config.mainScript.get.readSome.isEmpty) {
       infoOut("Could not read main script in the Viash config.")
       throw new ExitException(1)
     }

src/main/scala/io/viash/ViashTest.scala

@@ -240,7 +240,7 @@ object ViashTest extends Logging {
     val tests = conf.test_resources
 
     val testResults = tests.filter(_.isInstanceOf[Script]).map {
-      case test: Script if test.read.isEmpty =>
+      case test: Script if test.readSome.isEmpty =>
         TestOutput(test.filename, 1, "Test script does not exist.", "", 0L)
 
       case test: Script =>

src/main/scala/io/viash/config/Config.scala

@@ -553,7 +553,7 @@ case class Config(
       case s: Script => Some(s)
       case _ => None
     }
-  def mainCode: Option[String] = mainScript.flatMap(_.read)
+  def mainCode: Option[String] = mainScript.flatMap(_.readSome)
   // provide function to use resources.tail but that allows resources to be an empty list
   // If mainScript ends up being None because the first resource isn't a script, return the whole list
   def additionalResources = resources match {

src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala

@@ -47,10 +47,10 @@ trait ViashhubRepositoryTrait extends AbstractGitRepository {
   def getCacheIdentifier(): Option[String] =
     Some(s"viashhub-${fullRepo.replace("/", "-")}${tag.map(_.prepended('-')).getOrElse("")}")
 
-  val uri = s"https://viash-hub.com/$fullRepo.git"
-  lazy val uri_ssh = s"[email protected]:$fullRepo.git"
+  val uri = s"https://packages.viash-hub.com/$fullRepo.git"
+  lazy val uri_ssh = s"git@packages.viash-hub.com:$fullRepo.git"
   val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian
-  lazy val uri_nouser = s"https://${fakeCredentials}viash-hub.com/$fullRepo.git"
+  lazy val uri_nouser = s"https://${fakeCredentials}packages.viash-hub.com/$fullRepo.git"
 
   val storePath = fullRepo // no need to add 'viash-hub.com' to the store path as 'type' (vsh) will be added
 }

src/main/scala/io/viash/config/resources/Executable.scala

@@ -42,7 +42,7 @@ case class Executable(
 
   def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = ScriptInjectionMods()
 
-  override def read: Option[String] = None
+  override def readSome: Option[String] = None
 
   override def write(path: Path, overwrite: Boolean): Unit = {}
 

src/main/scala/io/viash/config/resources/Resource.scala

@@ -24,6 +24,7 @@ import io.viash.exceptions.MissingResourceFileException
 import java.nio.file.{Path, Paths}
 import java.nio.file.NoSuchFileException
 import io.viash.schemas._
+import java.io.FileNotFoundException
 
 @description(
   """Resources are files that support the component. The first resource should be @[a script](scripting_languages) that will be executed when the component is run. Additional resources will be copied to the same directory.
@@ -147,11 +148,17 @@ trait Resource {
     basenameRegex.replaceFirstIn(resourcePath, "")
   }
 
-  def read: Option[String] = {
-    if (text.isDefined) {
-      text
-    } else {
-      IO.readSome(uri.get)
+  def readSome: Option[String] = {
+    text.orElse(IO.readSome(uri.get))
+  }
+
+  def read: String = {
+    try {
+      text.getOrElse(IO.read(uri.get))
+    } catch {
+      case e: FileNotFoundException => 
+        val configString = parent.map(_.toString)
+        throw MissingResourceFileException.apply(uri.get.toString(), configString, e)
     }
   }
 
@@ -164,10 +171,7 @@ trait Resource {
       }
     } catch {
       case e: NoSuchFileException => 
-        val configString = parent match {
-          case Some(uri) => Some(uri.toString)
-          case _ => None
-        }
+        val configString = parent.map(_.toString)
         throw MissingResourceFileException.apply(path.toString(), configString, e)
     }
   }

src/main/scala/io/viash/config/resources/Script.scala

@@ -26,61 +26,59 @@ trait Script extends Resource {
 
   def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods
 
-  def readWithInjection(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): Option[String] = {
-    read.map(code => {
-      val lines = code.split("\n")
-      val startIndex = lines.indexWhere(_.contains("VIASH START"))
-      val endIndex = lines.indexWhere(_.contains("VIASH END"))
-
-      // compute mods
-      val mods = generateInjectionMods(argsMetaAndDeps, config)
-
-      val viashLines = Array(
-        companion.commentStr + " The following code has been auto-generated by Viash.",
-        mods.params
-      )
-
-      val li =
-        if (startIndex >= 0 && endIndex >= 0) {
-          val Whitespace = raw"^(\s+).*".r
-          val viashLinesWDelimiter = 
-            lines(startIndex) match {
-              case Whitespace(prefix) => 
-                viashLines.flatMap(_.split("\n")).map(prefix + _)
-              case _ => viashLines
-            }
-
-          lines.slice(0, startIndex + 1) ++ 
-          viashLinesWDelimiter ++ 
-          lines.slice(endIndex, lines.length)
-        } else {
-          Array(companion.commentStr + companion.commentStr + " VIASH START") ++
-          viashLines ++ 
-          Array(companion.commentStr + companion.commentStr + " VIASH END") ++
-          lines
-        }
-      val li2 = 
-        { if (mods.header.isEmpty()) Array.empty[String] else Array(mods.header) } ++
-        li ++
-        { if (mods.footer.isEmpty()) Array.empty[String] else Array(mods.footer) }
-
-      li2.mkString("\n")
-    })
+  def readWithInjection(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): String = {
+    val code = read
+    val lines = code.split("\n")
+    val startIndex = lines.indexWhere(_.contains("VIASH START"))
+    val endIndex = lines.indexWhere(_.contains("VIASH END"))
+
+    // compute mods
+    val mods = generateInjectionMods(argsMetaAndDeps, config)
+
+    val viashLines = Array(
+      companion.commentStr + " The following code has been auto-generated by Viash.",
+      mods.params
+    )
+
+    val li =
+      if (startIndex >= 0 && endIndex >= 0) {
+        val Whitespace = raw"^(\s+).*".r
+        val viashLinesWDelimiter = 
+          lines(startIndex) match {
+            case Whitespace(prefix) => 
+              viashLines.flatMap(_.split("\n")).map(prefix + _)
+            case _ => viashLines
+          }
+
+        lines.slice(0, startIndex + 1) ++ 
+        viashLinesWDelimiter ++ 
+        lines.slice(endIndex, lines.length)
+      } else {
+        Array(companion.commentStr + companion.commentStr + " VIASH START") ++
+        viashLines ++ 
+        Array(companion.commentStr + companion.commentStr + " VIASH END") ++
+        lines
+      }
+    val li2 = 
+      { if (mods.header.isEmpty()) Array.empty[String] else Array(mods.header) } ++
+      li ++
+      { if (mods.footer.isEmpty()) Array.empty[String] else Array(mods.footer) }
+
+    li2.mkString("\n")
   }
 
   def readWithoutInjection = {
-    read.map(code => {
-      val lines = code.split("\n")
-      val startIndex = lines.indexWhere(_.contains("VIASH START"))
-      val endIndex = lines.indexWhere(_.contains("VIASH END"))
-      val li = 
-        if (startIndex >= 0 && endIndex >= 0) {
-          lines.slice(0, startIndex + 1) ++ lines.slice(endIndex, lines.length)
-        } else {
-          lines
-        }
-      li.mkString("\n")
-    })
+    val code = read
+    val lines = code.split("\n")
+    val startIndex = lines.indexWhere(_.contains("VIASH START"))
+    val endIndex = lines.indexWhere(_.contains("VIASH END"))
+    val li = 
+      if (startIndex >= 0 && endIndex >= 0) {
+        lines.slice(0, startIndex + 1) ++ lines.slice(endIndex, lines.length)
+      } else {
+        lines
+      }
+    li.mkString("\n")
   }
 
   def command(script: String): String = (companion.executor :+ s"\"$script\"").mkString(" ")

src/main/scala/io/viash/engines/requirements/RRequirements.scala

@@ -87,15 +87,24 @@ case class RRequirements(
   @example("bioc_force_install: false", "yaml")
   @default("False")
   bioc_force_install: Boolean = false,
+
+  @description("Specifies whether to treat warnings as errors. Default: true.")
+  @example("warnings_as_errors: true", "yaml")
+  @default("True")
+  warnings_as_errors: Boolean = true,
 
   `type`: String = "r"
 ) extends Requirements {
-  assert(script.forall(!_.contains("'")), "R requirement '.script' field contains a single quote ('). This is not allowed.")
-
   def installCommands: List[String] = {
+    val prefix = if (warnings_as_errors) "options(warn = 2); " else ""
+
+    def runRCode(code: String): String = {
+      s"""Rscript -e '${prefix}${code.replaceAll("'", "'\"'\"'")}'"""
+    }
+
     val installRemotes =
       if ((packages ::: cran ::: git ::: github ::: gitlab ::: bitbucket ::: svn ::: url).nonEmpty) {
-        List("""Rscript -e 'if (!requireNamespace("remotes", quietly = TRUE)) install.packages("remotes")'""")
+        List(runRCode("""if (!requireNamespace("remotes", quietly = TRUE)) install.packages("remotes")"""))
       } else {
         Nil
       }
@@ -112,17 +121,17 @@ case class RRequirements(
 
     val installBiocManager =
       if (bioc.nonEmpty) {
-        List("""Rscript -e 'if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager")'""")
+        List(runRCode("""if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager")"""))
       } else {
         Nil
       }
     val installBioc =
       if (bioc.nonEmpty) {
         if (bioc_force_install) {
-          List(s"""Rscript -e 'BiocManager::install(c("${bioc.mkString("\", \"")}"))'""")
+          List(runRCode(s"""BiocManager::install(c("${bioc.mkString("\", \"")}"))"""))
         } else {
           bioc.map { biocPackage =>
-            s"""Rscript -e 'if (!requireNamespace("$biocPackage", quietly = TRUE)) BiocManager::install("$biocPackage")'"""
+            runRCode(s"""if (!requireNamespace("$biocPackage", quietly = TRUE)) BiocManager::install("$biocPackage")""")
           }
         }
       } else {
@@ -132,13 +141,13 @@ case class RRequirements(
     val installers = remotePairs.flatMap {
       case (_, Nil) => None
       case (str, list) =>
-        Some(s"""Rscript -e 'remotes::install_$str(c("${list.mkString("\", \"")}"), repos = "https://cran.rstudio.com")'""")
+        Some(runRCode(s"""remotes::install_$str(c("${list.mkString("\", \"")}"), repos = "https://cran.rstudio.com")"""))
     }
 
     val installScript =
       if (script.nonEmpty) {
         script.map { line =>
-          s"""Rscript -e '$line'"""
+          runRCode(line)
         }
       } else {
         Nil

src/main/scala/io/viash/runners/NextflowRunner.scala

@@ -217,7 +217,7 @@ final case class NextflowRunner(
       // if mainscript is a nextflow workflow
       case scr: NextflowScript =>
         s"""// user-provided Nextflow code
-          |${scr.readWithoutInjection.get.split("\n").mkString("\n|")}
+          |${scr.readWithoutInjection.split("\n").mkString("\n|")}
           |
           |// inner workflow hook
           |def innerWorkflowFactory(args) {

src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala

@@ -67,7 +67,7 @@ object NextflowHelper {
       includeMeta = true,
       filterInputs = true
     )
-    val code = res.readWithInjection(argsAndMeta, config).get
+    val code = res.readWithInjection(argsAndMeta, config)
     val escapedCode = Bash.escapeString(code, allowUnescape = true)
       .replace("\\", "\\\\")
       .replace("'''", "\\'\\'\\'")

src/main/scala/io/viash/wrapper/BashWrapper.scala

@@ -166,7 +166,7 @@ object BashWrapper {
 
       // if we want to debug our code
       case Some(res) if debugPath.isDefined =>
-        val code = res.readWithInjection(argsMetaAndDeps, config).get
+        val code = res.readWithInjection(argsMetaAndDeps, config)
         val escapedCode = Bash.escapeString(code, allowUnescape = true)
 
         s"""
@@ -178,7 +178,7 @@ object BashWrapper {
 
       // if mainResource is a script
       case Some(res) =>
-        val code = res.readWithInjection(argsMetaAndDeps, config).get
+        val code = res.readWithInjection(argsMetaAndDeps, config)
         val escapedCode = Bash.escapeString(code, allowUnescape = true)
 
         // check whether the script can be written to a temprorary location or

src/test/resources/testnextflowvdsl3/src/sub_workflow/config.vsh.yaml

@@ -6,6 +6,10 @@ arguments:
   - name: "--output"
     type: file
     direction: output
+  - name: "--required_int"
+    type: integer
+    direction: output
+    required: true
 resources:
   - type: nextflow_script
     path: main.nf