From 4dccbca650089746562ce9f7ef66263cb6c9cf06 Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Tue, 1 Mar 2022 10:26:59 +0100 Subject: [PATCH 01/11] Trap INT and SIGINT for (long-) running components --- .../dataintuitive/viash/wrapper/BashWrapper.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala b/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala index 51025623f..b238c7104 100644 --- a/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala +++ b/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala @@ -123,13 +123,18 @@ object BashWrapper { // check whether the script can be written to a temprorary location or // whether it needs to be a specific path - val scriptSetup = + val scriptSetup = s""" |tempscript=\\$$(mktemp "$$VIASH_TEMP/viash-run-${functionality.name}-XXXXXX") |function clean_up { | rm "\\$$tempscript" |} - |trap clean_up EXIT""".stripMargin + |function interrupt { + | echo -e "\\nCTRL-C Pressed..." + | exit 1 + |} + |trap clean_up EXIT + |trap interrupt INT SIGINT""".stripMargin val scriptPath = "\\$tempscript" s""" @@ -137,7 +142,8 @@ object BashWrapper { |cat > "$scriptPath" << 'VIASHMAIN' |$escapedCode |VIASHMAIN$cdToResources$resourcesToPath - |${res.meta.command(scriptPath)} + |${res.meta.command(scriptPath)} & + |wait |""".stripMargin // if we want to debug our code From 0257c50423f5bdc785783a38a5649be3dac6b275 Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Tue, 1 Mar 2022 11:42:03 +0100 Subject: [PATCH 02/11] Pass exit code correctly --- .../scala/com/dataintuitive/viash/wrapper/BashWrapper.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala b/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala index b238c7104..11475e8f7 100644 --- a/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala +++ b/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala @@ -143,7 +143,7 @@ object BashWrapper { |$escapedCode |VIASHMAIN$cdToResources$resourcesToPath |${res.meta.command(scriptPath)} & - |wait + |wait "\\$$!" |""".stripMargin // if we want to debug our code From 63893d889453cf99df4c56c4efb1e0874fe488d4 Mon Sep 17 00:00:00 2001 From: Toni Verbeiren Date: Tue, 1 Mar 2022 13:57:39 +0100 Subject: [PATCH 03/11] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0cfdc15..bde4ec5f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Viash 0.5.9 + +## NEW FEATURES + +* A long running Viash component can be interrupted by pressing CTRL-C or by sending it a `INT` or `SIGINT` signal. + # Viash 0.5.8 ## NEW FUNCTIONALITY From 9fe59144dbfedc74117515e8ed058a058988aad6 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Wed, 2 Mar 2022 11:20:01 +0100 Subject: [PATCH 04/11] print the whole error --- src/main/scala/com/dataintuitive/viash/Main.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/dataintuitive/viash/Main.scala b/src/main/scala/com/dataintuitive/viash/Main.scala index 1ecfcc80e..ec7f08c4c 100644 --- a/src/main/scala/com/dataintuitive/viash/Main.scala +++ b/src/main/scala/com/dataintuitive/viash/Main.scala @@ -35,7 +35,7 @@ object Main { internalMain(args) } catch { case e: Exception => - System.err.println(e.getMessage()) + System.err.println(e) System.exit(1) } } From 152f8f58c2cdbaf651a73cdb7c0b2151c6913789 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Fri, 11 Mar 2022 12:12:04 +0100 Subject: [PATCH 05/11] `viash ns list`: Added `--format yaml/json` argument --- CHANGELOG.md | 12 +++++++ .../com/dataintuitive/viash/CLIConf.scala | 14 ++++++++ .../scala/com/dataintuitive/viash/Main.scala | 7 ++-- .../com/dataintuitive/viash/ViashConfig.scala | 32 +++++++++++-------- .../dataintuitive/viash/ViashNamespace.scala | 4 +-- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0cfdc15..6bfd91853 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# Viash 0.5.9 + +## MINOR CHANGES + +* `viash ns list`: Added `--format yaml/json` argument to be able to return the + output as a json as well. Useful for when `jq` is installed but `yq` is not. Example: + ``` + viash ns list -p docker -f json | jq '.[] | .info.config' + ``` + +* `viash config view`: Same as above. + # Viash 0.5.8 ## NEW FUNCTIONALITY diff --git a/src/main/scala/com/dataintuitive/viash/CLIConf.scala b/src/main/scala/com/dataintuitive/viash/CLIConf.scala index 95b2981e8..3c738a66d 100644 --- a/src/main/scala/com/dataintuitive/viash/CLIConf.scala +++ b/src/main/scala/com/dataintuitive/viash/CLIConf.scala @@ -231,6 +231,13 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { | viash config view config.vsh.yaml | |Arguments:""".stripMargin) + val format = choice( + name = "format", + short = 'f', + default = Some("yaml"), + choices = List("yaml", "json"), + descr = "Which output format to use." + ) } val inject = new Subcommand("inject") with ViashCommand { banner( @@ -322,6 +329,13 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { | viash ns list [-n nmspc] [-s src] [-p docker] | |Arguments:""".stripMargin) + val format = choice( + name = "format", + short = 'f', + default = Some("yaml"), + choices = List("yaml", "json"), + descr = "Which output format to use." + ) } addSubcommand(build) diff --git a/src/main/scala/com/dataintuitive/viash/Main.scala b/src/main/scala/com/dataintuitive/viash/Main.scala index 1ecfcc80e..71bc9371e 100644 --- a/src/main/scala/com/dataintuitive/viash/Main.scala +++ b/src/main/scala/com/dataintuitive/viash/Main.scala @@ -88,16 +88,17 @@ object Main { append = cli.namespace.test.append() ) case List(cli.namespace, cli.namespace.list) => - val configs = readConfigs(cli.namespace.test, modifyFun = false) + val configs = readConfigs(cli.namespace.list, modifyFun = false) ViashNamespace.list( - configs = configs + configs = configs, + cli.namespace.list.format() ) case List(cli.config, cli.config.view) => val config = Config.readOnly( configPath = cli.config.view.config(), configMods = cli.config.view.config_mods() ) - ViashConfig.view(config) + ViashConfig.view(config, cli.config.view.format()) case List(cli.config, cli.config.inject) => val config = Config.readOnly( configPath = cli.config.inject.config(), diff --git a/src/main/scala/com/dataintuitive/viash/ViashConfig.scala b/src/main/scala/com/dataintuitive/viash/ViashConfig.scala index 0c92a2b19..c38b97e61 100644 --- a/src/main/scala/com/dataintuitive/viash/ViashConfig.scala +++ b/src/main/scala/com/dataintuitive/viash/ViashConfig.scala @@ -22,33 +22,39 @@ import com.dataintuitive.viash.helpers.IO import java.nio.file.{Files, Paths} import io.circe.syntax.EncoderOps -import io.circe.yaml.Printer +import io.circe.{Json, Printer => JsonPrinter} +import io.circe.yaml.{Printer => YamlPrinter} import scala.sys.process.Process import com.dataintuitive.viash.platforms.DebugPlatform object ViashConfig { - private val printer = Printer( + private val yamlPrinter = YamlPrinter( preserveOrder = true, dropNullKeys = true, - mappingStyle = Printer.FlowStyle.Block, + mappingStyle = YamlPrinter.FlowStyle.Block, splitLines = true, - stringStyle = Printer.StringStyle.DoubleQuoted + stringStyle = YamlPrinter.StringStyle.DoubleQuoted ) + private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) - def view(config: Config) { - val json = config.asJson - val configYamlStr = printer.pretty(json) - println(configYamlStr) + private def printJson(json: Json, format: String): Unit = { + val str = format match { + case "yaml" => yamlPrinter.pretty(json) + case "json" => jsonPrinter.print(json) + } + println(str) + } + + def view(config: Config, format: String): Unit = { + printJson(config.asJson, format) } - def viewMany(configs: List[Config]) { - val json = configs.asJson - val configYamlStr = printer.pretty(json) - println(configYamlStr) + def viewMany(configs: List[Config], format: String): Unit = { + printJson(configs.asJson, format) } - def inject(config: Config) { + def inject(config: Config): Unit = { val fun = config.functionality // check if config has a main script diff --git a/src/main/scala/com/dataintuitive/viash/ViashNamespace.scala b/src/main/scala/com/dataintuitive/viash/ViashNamespace.scala index bfcb28558..1196907af 100644 --- a/src/main/scala/com/dataintuitive/viash/ViashNamespace.scala +++ b/src/main/scala/com/dataintuitive/viash/ViashNamespace.scala @@ -159,7 +159,7 @@ object ViashNamespace { } } - def list(configs: List[Config]) { - ViashConfig.viewMany(configs) + def list(configs: List[Config], format: String = "yaml") { + ViashConfig.viewMany(configs, format) } } From 5a20c0a2ecdedf816a35a97a2c41a410ac73db87 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 12 Mar 2022 08:24:46 +0100 Subject: [PATCH 06/11] remove deprecated functionality --- CHANGELOG.md | 6 ++++++ src/main/scala/com/dataintuitive/viash/Main.scala | 4 ++-- .../com/dataintuitive/viash/platforms/DockerPlatform.scala | 7 +------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ac91a13..5d12e6866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ * `viash config view`: Same as above. +## DEPRECATION + +* `CLI`: Deprecated `-P` flag use `-p` intead. + +* `DockerPlatform`: Deprecated `version` value. + # Viash 0.5.8 ## NEW FUNCTIONALITY diff --git a/src/main/scala/com/dataintuitive/viash/Main.scala b/src/main/scala/com/dataintuitive/viash/Main.scala index 5cc47f858..cfbd9fdd1 100644 --- a/src/main/scala/com/dataintuitive/viash/Main.scala +++ b/src/main/scala/com/dataintuitive/viash/Main.scala @@ -116,7 +116,7 @@ object Main { ): Config = { Config.read( configPath = subcommand.config(), - platform = subcommand.platform.toOption | subcommand.platformid.toOption, + platform = subcommand.platform.toOption, modifyFun = modifyFun, configMods = subcommand.config_mods() ) @@ -133,7 +133,7 @@ object Main { val sourceDir = Paths.get(source) // create regex for filtering platform ids - val platformStr = (subcommand.platform.toOption | subcommand.platformid.toOption).getOrElse(".*") + val platformStr = (subcommand.platform.toOption).getOrElse(".*") // find *.vsh.* files and parse as config val scriptFiles = find(sourceDir, (path, attrs) => { diff --git a/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala b/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala index 8ed13653f..507dda6af 100644 --- a/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala +++ b/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala @@ -54,15 +54,10 @@ case class DockerPlatform( yum: Option[YumRequirements] = None, r: Option[RRequirements] = None, python: Option[PythonRequirements] = None, - docker: Option[DockerRequirements] = None, - - // deprecated - version: Option[Version] = None + docker: Option[DockerRequirements] = None ) extends Platform { override val hasSetup = true - assert(version.isEmpty, "docker platform: attribute 'version' is deprecated") - override val requirements: List[Requirements] = { val x = setup ::: From e9b17adc1ef3ae67ab28f1743f35f645b3211b0e Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 12 Mar 2022 09:19:02 +0100 Subject: [PATCH 07/11] `DockerPlatform`: Automatically add a few labels based on metadata to Dockerfile. --- CHANGELOG.md | 7 +++++++ .../viash/platforms/DockerPlatform.scala | 21 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d12e6866..7d5a1a88e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ * A long running Viash component can be interrupted by pressing CTRL-C or by sending it a `INT` or `SIGINT` signal. +* `DockerPlatform`: Automatically add a few labels based on metadata to Dockerfile. + +* `DockerPlatform`: Added value `target_image_source` for setting the source of the target image. Example: + ```yaml + target_image_source: https://github.com/foo/bar + ``` + ## MINOR CHANGES * `viash ns list`: Added `--format yaml/json` argument to be able to return the diff --git a/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala b/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala index 507dda6af..fc2fe3df0 100644 --- a/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala +++ b/src/main/scala/com/dataintuitive/viash/platforms/DockerPlatform.scala @@ -45,6 +45,7 @@ case class DockerPlatform( setup_strategy: DockerSetupStrategy = IfNeedBePullElseCachedBuild, privileged: Boolean = false, run_args: OneOrMore[String] = Nil, + target_image_source: Option[String] = None, oType: String = "docker", // setup variables @@ -133,8 +134,26 @@ case class DockerPlatform( } private def processDockerSetup(functionality: Functionality) = { + // construct labels from metadata + val auth = functionality.authors match { + case Nil => Nil + case aut: List[Author] => List(s"""authors="${aut.mkString(", ")}"""") + } + /*val descr = functionality.description.map{ des => + s"""org.opencontainers.image.description="${Bash.escape(des)}"""" + }.toList*/ + + val descr = List( + s"""org.opencontainers.image.description="Companion container for running component ${functionality.namespace.map(_ + " ").getOrElse("")}${functionality.name}"""" + ) + val imageSource = target_image_source.map(des => s"""org.opencontainers.image.source="${Bash.escape(des)}"""").toList + val labelReq = DockerRequirements(label = auth ::: descr ::: imageSource) + + val requirements2 = labelReq :: requirements + // get dependencies - val runCommands = requirements.flatMap(_.dockerCommands) + val runCommands = requirements2.flatMap(_.dockerCommands) + // don't draw defaults from functionality for the from image val fromImageInfo = Docker.getImageInfo( From 1f64f521d093be6c576d557304eba0aee5274f45 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 12 Mar 2022 09:23:13 +0100 Subject: [PATCH 08/11] commit unsaved changes --- .../scala/com/dataintuitive/viash/CLIConf.scala | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/main/scala/com/dataintuitive/viash/CLIConf.scala b/src/main/scala/com/dataintuitive/viash/CLIConf.scala index 3c738a66d..2dbcb2bd1 100644 --- a/src/main/scala/com/dataintuitive/viash/CLIConf.scala +++ b/src/main/scala/com/dataintuitive/viash/CLIConf.scala @@ -31,13 +31,6 @@ trait ViashCommand { "In addition, the path to a platform yaml file can also be specified.", required = false ) - val platformid = opt[String]( - short = 'P', - default = None, - descr = "[deprecated] passthrough option for platform.", - required = false, - hidden = true - ) val config = trailArg[String]( descr = "A viash config file (example: config.vsh.yaml). This argument can also be a script with the config as a header.", default = Some("config.vsh.yaml"), @@ -86,13 +79,6 @@ trait ViashNs { default = None, required = false ) - val platformid = opt[String]( - short = 'P', - descr = "[deprecated] passthrough option for platform.", - default = None, - required = false, - hidden = true - ) val parallel = opt[Boolean]( name = "parallel", short = 'l', From 5d9b4ba1cb0809cab537c03d6e0e06409f02fa2e Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 12 Mar 2022 09:25:13 +0100 Subject: [PATCH 09/11] update changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d5a1a88e..f02ee5e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,14 @@ ## NEW FEATURES -* A long running Viash component can be interrupted by pressing CTRL-C or by sending it a `INT` or `SIGINT` signal. +* `viash run`: A long running Viash component can be interrupted by pressing + CTRL-C or by sending it an `INT` or `SIGINT` signal. * `DockerPlatform`: Automatically add a few labels based on metadata to Dockerfile. -* `DockerPlatform`: Added value `target_image_source` for setting the source of the target image. Example: +* `DockerPlatform`: Added value `target_image_source` for setting the source of + the target image. This is used for defining labels in the dockerfile. + Example: ```yaml target_image_source: https://github.com/foo/bar ``` From 889466b7735f85fcbc37e40abac809179b70c73b Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 12 Mar 2022 09:25:56 +0100 Subject: [PATCH 10/11] bump version number --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index bb7d79997..f4da89acb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "viash" -version := "0.5.8" +version := "0.5.9" scalaVersion := "2.12.10" From bea32a63753afcb2b47e57c7a8549f017f0c13f3 Mon Sep 17 00:00:00 2001 From: Robrecht Cannoodt Date: Sat, 12 Mar 2022 09:39:39 +0100 Subject: [PATCH 11/11] fix unit test --- .../viash/auxiliary/MainBuildAuxiliaryDockerTag.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/scala/com/dataintuitive/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala b/src/test/scala/com/dataintuitive/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala index 30049c0c5..567ba17a0 100644 --- a/src/test/scala/com/dataintuitive/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala +++ b/src/test/scala/com/dataintuitive/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala @@ -42,10 +42,10 @@ class MainBuildAuxiliaryDockerTag extends FunSuite with BeforeAndAfterAll { val dockerout = Exec.run(Seq(executableBashTagFile.toString, "---dockerfile")) // we expect something basic like // FROM bash:5.0 - // + // LABEL ... // RUN : // Allow for extra spaces just in case the format changes slightly format-wise but without functional differences - val regex = """^FROM bash:5\.0[\r\n\s]*RUN\s+:\s*$""".r + val regex = """^FROM bash:5\.0[\r\n\s]*.*""".r assert(regex.findFirstIn(dockerout).isDefined, regex.toString) } @@ -75,10 +75,10 @@ class MainBuildAuxiliaryDockerTag extends FunSuite with BeforeAndAfterAll { val dockerout = Exec.run(Seq(executableBashTagFile.toString, "---dockerfile")) // we expect something basic like // FROM bash:3.2 - // + // LABEL ... // RUN : // Allow for extra spaces just in case the format changes slightly format-wise but without functional differences - val regex = """^FROM bash:3\.2[\r\n\s]*RUN\s+:\s*$""".r + val regex = """^FROM bash:3\.2[\r\n\s]*.*""".r assert(regex.findFirstIn(dockerout).isDefined, regex.toString) }