Merge branch 'develop' into feature/replayable_multi_output_stream

CHANGELOG.md

@@ -10,9 +10,23 @@ TODO add summary
   Additionally, the `native platform` and `docker platform` became a `executable runner`, `nextflow platform` became a `nextflow runner`.
   The fields of `docker platform` is split between `docker engine` and `docker runner`: `port`, `workdir`, `setup_strategy`, and `run_args` are captured by the `runner` as they define how the component is run. The other fields are captured by the `engine` as they define the environment in which the component is run. One exception is `chown` which is rarely set to false and is now always enabled.
 
+## NEW FUNCTIONALITY
+
+* `export json_schema`: Add a `--strict` option to output a subset of the schema representing the internal structure of the Viash config (PR #564).
+
+* `config view` and `ns list`: Do not output internal functionality fields (#564). Additionally, add a validation that no internal fields are present when reading a Viash config file.
+
 ## MINOR CHANGES
 
+* `testbenches`: Add testbenches for local dependencies (PR #565).
+
+* `testbenches`: Refactor testbenches helper functions to uniformize them (PR #565).
+
 * `logging`: Preserve log order of StdOut and StdErr messages during reading configs in namespaces (PR #571).
+
+## BUG FIXES
+
+* `__merge__`: Handle invalid yaml during merging (PR #570). There was not enough error handling during this operation. Switched to the more advanced `Convert.textToJson` helper method.
 
 # Viash 0.8.0-RC5 (2023-10-11): Fix run workflow
 

codecov.yml

@@ -0,0 +1,12 @@
+coverage:
+  precision: 2
+  round: down
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 1%
+    patch:
+      default:
+        enabled: yes
+        threshold: 5%

src/main/scala/io/viash/Main.scala

@@ -368,7 +368,9 @@ object Main extends Logging {
         val output = cli.export.json_schema.output.toOption.map(Paths.get(_))
         ViashExport.exportJsonSchema(
           output,
-          format = cli.export.json_schema.format()
+          format = cli.export.json_schema.format(),
+          strict = cli.export.json_schema.strict(),
+          minimal = cli.export.json_schema.minimal()
         )
         0
       case List(cli.export, cli.export.resource) =>

src/main/scala/io/viash/ViashExport.scala

@@ -65,8 +65,8 @@ object ViashExport extends Logging {
   }
 
 
-  def exportJsonSchema(output: Option[Path], format: String): Unit = {
-    val data = JsonSchema.getJsonSchema
+  def exportJsonSchema(output: Option[Path], format: String, strict: Boolean, minimal: Boolean): Unit = {
+    val data = JsonSchema.getJsonSchema(strict, minimal)
     val str = data.toFormattedString(format)
     if (output.isDefined) {
       Files.write(output.get, str.getBytes())

src/main/scala/io/viash/cli/CLIConf.scala

@@ -542,6 +542,16 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin
         choices = List("yaml", "json"),
         descr = "Which output format to use."
       )
+      val strict = registerOpt[Boolean](
+        name = "strict",
+        default = Some(false),
+        descr = "Whether or not to use export the strict schema variant."
+      )
+      val minimal = registerOpt[Boolean](
+        name = "minimal",
+        default = Some(false),
+        descr = "Whether or not to output extra schema annotations."
+      )
     }
 
     addSubcommand(resource)

src/main/scala/io/viash/config/Config.scala

@@ -86,7 +86,7 @@ case class Config(
   @default("Empty")
   engines: List[Engine] = Nil,
 
-  @internalFunctionality
+  @undocumented
   info: Option[Info] = None
 ) {
 
@@ -281,33 +281,33 @@ object Config extends Logging {
     // convert Json into Config
     val conf0 = Convert.jsonToClass[Config](json2, uri.toString())
 
-    /* CONFIG 1: store parent path in resource to be able to access them in the future */
-    val parentURI = uri.resolve("")
-    val resources = conf0.functionality.resources.map(_.copyWithAbsolutePath(parentURI, projectDir))
-    val tests = conf0.functionality.test_resources.map(_.copyWithAbsolutePath(parentURI, projectDir))
-
-    // copy resources with updated paths into config and return
-    val conf1 = conf0.copy(
-      functionality = conf0.functionality.copy(
-        resources = resources,
-        test_resources = tests
-      )
-    )
-
-    /* CONFIG 2: apply post-parse config mods */
+    /* CONFIG 1: apply post-parse config mods */
     // apply config mods only if need be
-    val conf2 = 
+    val conf1 = 
       if (confMods.postparseCommands.nonEmpty) {
         // turn config back into json
-        val js = encodeConfig(conf1)
+        val js = encodeConfig(conf0)
         // apply config mods
         val modifiedJs = confMods(js, preparse = false)
         // turn json back into a config
         modifiedJs.as[Config].fold(throw _, identity)
       } else {
-        conf1
+        conf0
       }
 
+    /* CONFIG 1: store parent path in resource to be able to access them in the future */
+    val parentURI = uri.resolve("")
+    val resources = conf1.functionality.resources.map(_.copyWithAbsolutePath(parentURI, projectDir))
+    val tests = conf1.functionality.test_resources.map(_.copyWithAbsolutePath(parentURI, projectDir))
+
+    // copy resources with updated paths into config and return
+    val conf2 = conf1.copy(
+      functionality = conf0.functionality.copy(
+        resources = resources,
+        test_resources = tests
+      )
+    )
+
     /* CONFIG 3: add info */
     // gather git info
     // todo: resolve git in project?

src/main/scala/io/viash/config/package.scala

@@ -27,11 +27,12 @@ package object config {
   import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
   import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._
   import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
+  import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
 
   implicit val customConfig: Configuration = Configuration.default.withDefaults
 
   // encoders and decoders for Config
-  implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoder
+  implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config]
   implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config].prepare{
     checkDeprecation[Config](_)
     // map platforms to runners and engines

src/main/scala/io/viash/engines/NativeEngine.scala

@@ -30,7 +30,7 @@ import io.viash.schemas._
   "yaml")
 @subclass("native")
 final case class NativeEngine(
-  @description("Name of the engine. As with all engines, you can give a engine a different name. By specifying `id: foo`, you can target this engine (only) by specifying `...` in any of the Viash commands.")
+  @description("Name of the engine. As with all engines, you can give an engine a different name. By specifying `id: foo`, you can target this engine (only) by specifying `...` in any of the Viash commands.")
   @example("id: foo", "yaml")
   @default("native")
   id: String = "native",

src/main/scala/io/viash/exceptions/ConfigException.scala

@@ -43,4 +43,9 @@ case class ConfigParserSubTypeException(tpe: String, validTypes: List[String], j
 case class ConfigParserValidationException(tpe: String, json: String) extends Exception {
   val shortType = tpe.split("\\.").last
   override def getMessage(): String = s"Invalid data fields for $shortType.\n$json"
+}
+
+case class ConfigParserMergeException(uri: String, innerMessage: String, json: String) extends AbstractConfigException {
+  val e: Throwable = null
+  override def getMessage(): String = json
 }

src/main/scala/io/viash/functionality/arguments/package.scala

@@ -22,6 +22,7 @@ import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfigur
 import cats.syntax.functor._ // for .widen
 import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
 import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
+import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
 import io.viash.exceptions.ConfigParserSubTypeException
 
 package object arguments {
@@ -62,14 +63,14 @@ package object arguments {
   }
 
   // encoders and decoders for Argument
-  implicit val encodeStringArgument: Encoder.AsObject[StringArgument] = deriveConfiguredEncoder
-  implicit val encodeIntegerArgument: Encoder.AsObject[IntegerArgument] = deriveConfiguredEncoder
-  implicit val encodeLongArgument: Encoder.AsObject[LongArgument] = deriveConfiguredEncoder
-  implicit val encodeDoubleArgument: Encoder.AsObject[DoubleArgument] = deriveConfiguredEncoder
-  implicit val encodeBooleanArgumentR: Encoder.AsObject[BooleanArgument] = deriveConfiguredEncoder
-  implicit val encodeBooleanArgumentT: Encoder.AsObject[BooleanTrueArgument] = deriveConfiguredEncoder
-  implicit val encodeBooleanArgumentF: Encoder.AsObject[BooleanFalseArgument] = deriveConfiguredEncoder
-  implicit val encodeFileArgument: Encoder.AsObject[FileArgument] = deriveConfiguredEncoder
+  implicit val encodeStringArgument: Encoder.AsObject[StringArgument] = deriveConfiguredEncoderStrict[StringArgument]
+  implicit val encodeIntegerArgument: Encoder.AsObject[IntegerArgument] = deriveConfiguredEncoderStrict[IntegerArgument]
+  implicit val encodeLongArgument: Encoder.AsObject[LongArgument] = deriveConfiguredEncoderStrict[LongArgument]
+  implicit val encodeDoubleArgument: Encoder.AsObject[DoubleArgument] = deriveConfiguredEncoderStrict[DoubleArgument]
+  implicit val encodeBooleanArgumentR: Encoder.AsObject[BooleanArgument] = deriveConfiguredEncoderStrict[BooleanArgument]
+  implicit val encodeBooleanArgumentT: Encoder.AsObject[BooleanTrueArgument] = deriveConfiguredEncoderStrict[BooleanTrueArgument]
+  implicit val encodeBooleanArgumentF: Encoder.AsObject[BooleanFalseArgument] = deriveConfiguredEncoderStrict[BooleanFalseArgument]
+  implicit val encodeFileArgument: Encoder.AsObject[FileArgument] = deriveConfiguredEncoderStrict[FileArgument]
 
   implicit def encodeArgument[A <: Argument[_]]: Encoder[A] = Encoder.instance {
     par =>

src/main/scala/io/viash/functionality/dependencies/package.scala

@@ -25,13 +25,14 @@ import cats.syntax.functor._
 package object dependencies {
 
   import io.viash.helpers.circe._
+  import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
 
   // encoders and decoders for Argument
-  implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoder
-  implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoder
-  implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoder
-  implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoder
-  implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoder
+  implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict
+  implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict
+  implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict
+  implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoderStrict
+  implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoderStrict
   implicit def encodeRepository[A <: Repository]: Encoder[A] = Encoder.instance {
     par =>
       val typeJson = Json.obj("type" -> Json.fromString(par.`type`))

src/main/scala/io/viash/functionality/package.scala

@@ -34,9 +34,10 @@ package object functionality extends Logging {
   import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
   import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._
   import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
+  import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
 
   // encoder and decoder for Functionality
-  implicit val encodeFunctionality: Encoder.AsObject[Functionality] = deriveConfiguredEncoder
+  implicit val encodeFunctionality: Encoder.AsObject[Functionality] = deriveConfiguredEncoderStrict[Functionality]
 
   // add file & direction defaults for inputs & outputs
   implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderFullChecks

src/main/scala/io/viash/functionality/resources/package.scala

@@ -28,6 +28,7 @@ package object resources {
 
   import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
   import io.viash.helpers.circe._
+  import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
 
   implicit val encodeURI: Encoder[URI] = Encoder.instance {
     uri => Json.fromString(uri.toString)
@@ -37,15 +38,15 @@ package object resources {
   }
 
   // encoders and decoders for Object
-  implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoder
-  implicit val encodePythonScript: Encoder.AsObject[PythonScript] = deriveConfiguredEncoder
-  implicit val encodeRScript: Encoder.AsObject[RScript] = deriveConfiguredEncoder
-  implicit val encodeJavaScriptScript: Encoder.AsObject[JavaScriptScript] = deriveConfiguredEncoder
-  implicit val encodeNextflowScript: Encoder.AsObject[NextflowScript] = deriveConfiguredEncoder
-  implicit val encodeScalaScript: Encoder.AsObject[ScalaScript] = deriveConfiguredEncoder
-  implicit val encodeCSharpScript: Encoder.AsObject[CSharpScript] = deriveConfiguredEncoder
-  implicit val encodeExecutable: Encoder.AsObject[Executable] = deriveConfiguredEncoder
-  implicit val encodePlainFile: Encoder.AsObject[PlainFile] = deriveConfiguredEncoder
+  implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoderStrict[BashScript]
+  implicit val encodePythonScript: Encoder.AsObject[PythonScript] = deriveConfiguredEncoderStrict[PythonScript]
+  implicit val encodeRScript: Encoder.AsObject[RScript] = deriveConfiguredEncoderStrict[RScript]
+  implicit val encodeJavaScriptScript: Encoder.AsObject[JavaScriptScript] = deriveConfiguredEncoderStrict[JavaScriptScript]
+  implicit val encodeNextflowScript: Encoder.AsObject[NextflowScript] = deriveConfiguredEncoderStrict[NextflowScript]
+  implicit val encodeScalaScript: Encoder.AsObject[ScalaScript] = deriveConfiguredEncoderStrict[ScalaScript]
+  implicit val encodeCSharpScript: Encoder.AsObject[CSharpScript] = deriveConfiguredEncoderStrict[CSharpScript]
+  implicit val encodeExecutable: Encoder.AsObject[Executable] = deriveConfiguredEncoderStrict[Executable]
+  implicit val encodePlainFile: Encoder.AsObject[PlainFile] = deriveConfiguredEncoderStrict[PlainFile]
 
   implicit def encodeResource[A <: Resource]: Encoder[A] = Encoder.instance {
     par =>

src/main/scala/io/viash/helpers/DependencyResolver.scala

@@ -28,13 +28,13 @@ import java.nio.file.Files
 import java.io.IOException
 import java.io.UncheckedIOException
 import io.viash.helpers.{IO, Logging}
-import io.circe.yaml.parser
 import io.circe.Json
 import io.viash.config.Config._
 import io.viash.ViashNamespace
 import io.viash.functionality.dependencies.Dependency
 import io.viash.functionality.resources.NextflowScript
 import io.viash.exceptions.MissingDependencyException
+import io.viash.helpers.circe.Convert
 
 object DependencyResolver extends Logging {
 
@@ -227,7 +227,7 @@ object DependencyResolver extends Logging {
       // The yaml file in the target folder should be final
       // We're also assuming that the file will be proper yaml and an actual viash config file
       val yamlText = IO.read(IO.uri(configPath))
-      val json = parser.parse(yamlText).toOption.get
+      val json = Convert.textToJson(yamlText, configPath)
 
       def getFunctionalityName(json: Json): Option[String] = {
         json.hcursor.downField("functionality").downField("name").as[String].toOption
@@ -256,7 +256,7 @@ object DependencyResolver extends Logging {
   def getSparseDependencyInfo(configPath: String): List[String] = {
     try {
       val yamlText = IO.read(IO.uri(configPath))
-      val json = parser.parse(yamlText).toOption.get
+      val json = Convert.textToJson(yamlText, configPath)
 
       val dependencies = json.hcursor.downField("functionality").downField("dependencies").focus.flatMap(_.asArray).get
       dependencies.flatMap(_.hcursor.downField("writtenPath").as[String].toOption).toList

src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala

@@ -28,45 +28,52 @@ import io.viash.schemas.ParameterSchema
 import io.circe.ACursor
 
 import io.viash.helpers.Logging
+import io.viash.schemas.CollectedSchemas
 
 object DeriveConfiguredDecoderWithDeprecationCheck extends Logging {
 
-  private def memberDeprecationCheck(name: String, history: List[CursorOp], T: Type): Unit = {
-    val m = T.member(TermName(name))
-    val schema = ParameterSchema(name, "", List.empty, m.annotations)
-    val deprecated = schema.flatMap(_.deprecated)
-    val removed = schema.flatMap(_.removed)
-    if (deprecated.isDefined) {
-      val d = deprecated.get
-      val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".")
-      info(s"Warning: .$historyString.$name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.")
+  private def memberDeprecationCheck(name: String, history: List[CursorOp], parameters: List[ParameterSchema]): Unit = {
+    val schema = parameters.find(p => p.name == name).getOrElse(ParameterSchema("", "", "", None, None, None, None, None, None, None, None, false, false))
+
+    lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".")
+
+    schema.deprecated match {
+      case Some(d) =>
+        info(s"Warning: .$historyString.$name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.")
+      case _ =>
+    }
+    schema.removed match {
+      case Some(r) => 
+        info(s"Error: .$historyString.$name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.")
+      case _ =>
     }
-    if (removed.isDefined) {
-      val r = removed.get
-      val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".")
-      info(s"Error: .$historyString.$name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.")
+    if (schema.hasInternalFunctionality) {
+      error(s"Error: .$historyString.$name is internal functionality.")
+      throw new RuntimeException(s"Internal functionality used: .$historyString.$name")
     }
   }
 
-  private def selfDeprecationCheck(T: Type): Unit = {
-    val baseClass = T.baseClasses.head
-    val name = baseClass.fullName.split('.').last
-    val schema = ParameterSchema("", "", List.empty, baseClass.annotations)
-    val deprecated = schema.flatMap(_.deprecated)
-    val removed = schema.flatMap(_.removed)
-    if (deprecated.isDefined) {
-      val d = deprecated.get
-      info(s"Warning: $name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.")
+  private def selfDeprecationCheck(parameters: List[ParameterSchema]): Unit = {
+    val schema = parameters.find(p => p.name == "__this__").get
+
+    schema.deprecated match {
+      case Some(d) =>
+        info(s"Warning: ${schema.`type`} is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.")
+      case _ =>
     }
-    if (removed.isDefined) {
-      val r = removed.get
-      info(s"Error: $name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.")
+    schema.removed match {
+      case Some(r) =>
+        info(s"Error: ${schema.`type`} was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.")
+      case _ =>
     }
   }
 
   // 
   def checkDeprecation[A](cursor: ACursor)(implicit tag: TypeTag[A]) : ACursor = {
-    selfDeprecationCheck(typeOf[A])
+    val parameters = CollectedSchemas.getParameters[A]()
+
+    selfDeprecationCheck(parameters)
+
     // check each defined 'key' value
     for (key <- cursor.keys.getOrElse(Nil)) {
       val isEmpty = 
@@ -77,7 +84,7 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging {
           case _ => false
         }
       if (!isEmpty) {
-        memberDeprecationCheck(key, cursor.history, typeOf[A])
+        memberDeprecationCheck(key, cursor.history, parameters)
       }
     }
     cursor // return unchanged json info