From d19a0d313fc64b94de020c306158275fa70ea96b Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 8 Aug 2024 16:43:29 +0200 Subject: [PATCH 01/46] Disable any reflection & circe calls as it was tripping over its toes fix a range of issues until it compiles. This won't work though, too much stuff bypassed. --- build.sbt | 23 +- src/main/scala/io/viash/Main.scala | 37 +-- src/main/scala/io/viash/ViashTest.scala | 2 +- src/main/scala/io/viash/cli/CLIConf.scala | 46 ++-- .../io/viash/cli/DocumentedSubcommand.scala | 14 +- src/main/scala/io/viash/cli/package.scala | 3 +- src/main/scala/io/viash/config/Config.scala | 13 +- .../scala/io/viash/config/ConfigMeta.scala | 2 +- .../scala/io/viash/config/References.scala | 2 +- .../io/viash/config/arguments/package.scala | 82 +++--- .../dependencies/AbstractGitRepository.scala | 2 +- .../dependencies/GithubRepositoryTrait.scala | 2 +- .../ViashhubRepositoryTrait.scala | 2 +- .../viash/config/dependencies/package.scala | 148 +++++------ src/main/scala/io/viash/config/package.scala | 58 +++-- .../viash/config/resources/PythonScript.scala | 58 ++--- .../viash/config/resources/ScalaScript.scala | 80 +++--- .../io/viash/config/resources/package.scala | 158 ++++++------ .../viash/config_mods/ConfigModParser.scala | 4 +- .../io/viash/config_mods/ConfigMods.scala | 2 +- .../scala/io/viash/engines/DockerEngine.scala | 1 + .../io/viash/engines/docker/package.scala | 26 +- src/main/scala/io/viash/engines/package.scala | 60 ++--- .../viash/engines/requirements/package.scala | 81 +++--- .../viash/functionality/Functionality.scala | 2 +- .../io/viash/functionality/package.scala | 12 +- src/main/scala/io/viash/helpers/Git.scala | 2 +- src/main/scala/io/viash/helpers/Helper.scala | 1 + src/main/scala/io/viash/helpers/IO.scala | 2 +- .../DeriveConfiguredDecoderFullChecks.scala | 26 +- ...onfiguredDecoderWithDeprecationCheck.scala | 144 +++++------ ...ConfiguredDecoderWithValidationCheck.scala | 84 +++--- .../circe/DeriveConfiguredEncoderStrict.scala | 28 +- .../io/viash/helpers/circe/RichJson.scala | 2 +- .../io/viash/helpers/circe/package.scala | 8 +- .../io/viash/lenses/AppliedConfigLenses.scala | 16 +- .../scala/io/viash/lenses/ConfigLenses.scala | 4 +- .../viash/packageConfig/PackageConfig.scala | 3 +- .../io/viash/packageConfig/package.scala | 8 +- .../scala/io/viash/platforms/package.scala | 39 +-- .../io/viash/runners/NextflowRunner.scala | 2 +- .../io/viash/runners/executable/package.scala | 26 +- .../runners/nextflow/NextflowConfig.scala | 1 + .../runners/nextflow/NextflowHelper.scala | 6 +- .../io/viash/runners/nextflow/package.scala | 18 +- src/main/scala/io/viash/runners/package.scala | 61 ++--- .../io/viash/schemas/CollectedSchemas.scala | 153 +++++------ .../scala/io/viash/schemas/JsonSchema.scala | 243 +++++++++--------- .../io/viash/schemas/ParameterSchema.scala | 228 ++++++++-------- .../scala/io/viash/wrapper/BashWrapper.scala | 21 +- 50 files changed, 1045 insertions(+), 1001 deletions(-) diff --git a/build.sbt b/build.sbt index d1a66c267..4746bd264 100644 --- a/build.sbt +++ b/build.sbt @@ -2,31 +2,36 @@ name := "viash" version := "0.9.0-dev" -scalaVersion := "2.13.12" +scalaVersion := "3.3.3" libraryDependencies ++= Seq( "org.scalactic" %% "scalactic" % "3.2.15" % "test", "org.scalatest" %% "scalatest" % "3.2.15" % "test", "org.rogach" %% "scallop" % "5.0.0", - "org.scala-lang" % "scala-reflect" % scalaVersion.value, + // "org.scala-lang" % "scala-reflect" % scalaVersion.value, "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1", "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", - "com.github.julien-truffaut" %% "monocle-core" % "2.1.0", - "com.github.julien-truffaut" %% "monocle-macro" % "2.1.0" + "dev.optics" %% "monocle-core" % "3.1.0", + "dev.optics" %% "monocle-macro" % "3.1.0" ) -val circeVersion = "0.14.1" +val circeVersion = "0.14.9" libraryDependencies ++= Seq( "io.circe" %% "circe-core", "io.circe" %% "circe-generic", "io.circe" %% "circe-parser", - "io.circe" %% "circe-generic-extras", - "io.circe" %% "circe-optics", - "io.circe" %% "circe-yaml" + // "io.circe" %% "circe-generic-extras", + // "io.circe" %% "circe-optics", + // "io.circe" %% "circe-yaml" ).map(_ % circeVersion) -scalacOptions ++= Seq("-unchecked", "-deprecation") +libraryDependencies ++= Seq( + "io.circe" %% "circe-optics" % "0.15.0", + "io.circe" %% "circe-yaml" % "0.16.0", +) + +scalacOptions ++= Seq("-unchecked", "-deprecation", "-explain") organization := "Data Intuitive" startYear := Some(2020) diff --git a/src/main/scala/io/viash/Main.scala b/src/main/scala/io/viash/Main.scala index 5945a4213..79909f47f 100644 --- a/src/main/scala/io/viash/Main.scala +++ b/src/main/scala/io/viash/Main.scala @@ -34,6 +34,7 @@ import io.viash.helpers.LoggerLevel import io.viash.runners.Runner import io.viash.config.AppliedConfig import io.viash.engines.Engine +import io.viash.helpers.data_structures.* object Main extends Logging { private val pkg = getClass.getPackage @@ -387,40 +388,40 @@ object Main extends Logging { ) ViashConfig.inject(config.config) 0 - case List(cli.export, cli.export.cli_schema) => - val output = cli.export.cli_schema.output.toOption.map(Paths.get(_)) + case List(cli.`export`, cli.`export`.cli_schema) => + val output = cli.`export`.cli_schema.output.toOption.map(Paths.get(_)) ViashExport.exportCLISchema( output, - format = cli.export.cli_schema.format() + format = cli.`export`.cli_schema.format() ) 0 - case List(cli.export, cli.export.cli_autocomplete) => - val output = cli.export.cli_autocomplete.output.toOption.map(Paths.get(_)) + case List(cli.`export`, cli.`export`.cli_autocomplete) => + val output = cli.`export`.cli_autocomplete.output.toOption.map(Paths.get(_)) ViashExport.exportAutocomplete( output, - format = cli.export.cli_autocomplete.format() + format = cli.`export`.cli_autocomplete.format() ) 0 - case List(cli.export, cli.export.config_schema) => - val output = cli.export.config_schema.output.toOption.map(Paths.get(_)) + case List(cli.`export`, cli.`export`.config_schema) => + val output = cli.`export`.config_schema.output.toOption.map(Paths.get(_)) ViashExport.exportConfigSchema( output, - format = cli.export.config_schema.format() + format = cli.`export`.config_schema.format() ) 0 - case List(cli.export, cli.export.json_schema) => - val output = cli.export.json_schema.output.toOption.map(Paths.get(_)) + case List(cli.`export`, cli.`export`.json_schema) => + val output = cli.`export`.json_schema.output.toOption.map(Paths.get(_)) ViashExport.exportJsonSchema( output, - format = cli.export.json_schema.format(), - strict = cli.export.json_schema.strict(), - minimal = cli.export.json_schema.minimal() + format = cli.`export`.json_schema.format(), + strict = cli.`export`.json_schema.strict(), + minimal = cli.`export`.json_schema.minimal() ) 0 - case List(cli.export, cli.export.resource) => - val output = cli.export.resource.output.toOption.map(Paths.get(_)) + case List(cli.`export`, cli.`export`.resource) => + val output = cli.`export`.resource.output.toOption.map(Paths.get(_)) ViashExport.exportResource( - cli.export.resource.path.toOption.get, + cli.`export`.resource.path.toOption.get, output ) 0 @@ -551,7 +552,7 @@ object Main extends Logging { configs0 } - configs1 + configs1 } // Handle dependencies operations for a single config diff --git a/src/main/scala/io/viash/ViashTest.scala b/src/main/scala/io/viash/ViashTest.scala index 8c6b9deed..9026a8d2d 100644 --- a/src/main/scala/io/viash/ViashTest.scala +++ b/src/main/scala/io/viash/ViashTest.scala @@ -283,7 +283,7 @@ object ViashTest extends Logging { val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir)) // assemble full resources list for test - val confFinal = resourcesLens.set( + val confFinal = resourcesLens.replace( testBash :: // the executable, wrapped with an executable runner, // to be run inside of the runner of the test diff --git a/src/main/scala/io/viash/cli/CLIConf.scala b/src/main/scala/io/viash/cli/CLIConf.scala index b9ca4049e..a8245a1de 100644 --- a/src/main/scala/io/viash/cli/CLIConf.scala +++ b/src/main/scala/io/viash/cli/CLIConf.scala @@ -23,7 +23,7 @@ import io.viash.exceptions.ExitException import io.viash.helpers.Logging trait ViashCommand { - _: DocumentedSubcommand => + this: DocumentedSubcommand => val platform = registerOpt[String]( name = "platform", short = Some('p'), @@ -69,7 +69,7 @@ trait ViashCommand { ) } trait ViashRunner { - _: DocumentedSubcommand => + this: DocumentedSubcommand => val cpus = registerOpt[Int]( name = "cpus", default = None, @@ -84,7 +84,7 @@ trait ViashRunner { ) } trait ViashNs { - _: DocumentedSubcommand => + this: DocumentedSubcommand => val query = registerOpt[String]( name = "query", short = Some('q'), @@ -158,7 +158,7 @@ trait ViashNs { } trait ViashNsBuild { - _: DocumentedSubcommand => + this: DocumentedSubcommand => val target = registerOpt[String]( name = "target", short = Some('t'), @@ -167,7 +167,7 @@ trait ViashNsBuild { ) } trait WithTemporary { - _: DocumentedSubcommand => + this: DocumentedSubcommand => val keep = registerOpt[String]( name = "keep", short = Some('k'), @@ -179,7 +179,7 @@ trait WithTemporary { } trait ViashLogger { - _: DocumentedSubcommand => + this: DocumentedSubcommand => val colorize = registerChoice( name = "colorize", short = None, @@ -250,7 +250,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin | viash run config.vsh.yaml""".stripMargin) } - val build = new DocumentedSubcommand("build") with ViashCommand with ViashLogger { + object build extends DocumentedSubcommand("build") with ViashCommand with ViashLogger { banner( "viash build", "Build an executable from the provided viash config file.", @@ -276,7 +276,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val test = new DocumentedSubcommand("test") with ViashCommand with WithTemporary with ViashRunner with ViashLogger { + object test extends DocumentedSubcommand("test") with ViashCommand with WithTemporary with ViashRunner with ViashLogger { banner( "viash test", "Test the component using the tests defined in the viash config file.", @@ -308,8 +308,8 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin | viash run meta.vsh.yaml""".stripMargin) } - val config = new DocumentedSubcommand("config") { - val view = new DocumentedSubcommand("view") with ViashCommand with ViashLogger { + object config extends DocumentedSubcommand("config") { + object view extends DocumentedSubcommand("view") with ViashCommand with ViashLogger { banner( "viash config view", "View the config file after parsing.", @@ -328,7 +328,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin descr = "DEPRECATED. This is now always enabled. Whether or not to postprocess each component's @[argument groups](argument_groups)." ) } - val inject = new DocumentedSubcommand("inject") with ViashCommand with ViashLogger { + object inject extends DocumentedSubcommand("inject") with ViashCommand with ViashLogger { banner( "viash config inject", "Inject a Viash header into the main script of a Viash component.", @@ -342,9 +342,9 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin shortSubcommandsHelp(true) } - val namespace = new DocumentedSubcommand("ns") { + object namespace extends DocumentedSubcommand("ns") { - val build = new DocumentedSubcommand("build") with ViashNs with ViashNsBuild with ViashLogger { + object build extends DocumentedSubcommand("build") with ViashNs with ViashNsBuild with ViashLogger { banner( "viash ns build", "Build a namespace from many viash config files.", @@ -368,7 +368,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val test = new DocumentedSubcommand("test") with ViashNs with WithTemporary with ViashRunner with ViashLogger { + object test extends DocumentedSubcommand("test") with ViashNs with WithTemporary with ViashRunner with ViashLogger { banner( "viash ns test", "Test a namespace containing many viash config files.", @@ -405,7 +405,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val list = new DocumentedSubcommand("list") with ViashNs with ViashLogger { + object list extends DocumentedSubcommand("list") with ViashNs with ViashLogger { banner( "viash ns list", "List a namespace containing many viash config files.", @@ -425,7 +425,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val exec = new DocumentedSubcommand("exec") with ViashNs with ViashLogger { + object exec extends DocumentedSubcommand("exec") with ViashNs with ViashLogger { banner( "viash ns exec", """Execute a command for all found Viash components. @@ -505,10 +505,10 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin shortSubcommandsHelp(true) } - val `export` = new DocumentedSubcommand("export") { + object `export` extends DocumentedSubcommand("export") { hidden = true - val resource = new DocumentedSubcommand("resource") with ViashLogger { + object resource extends DocumentedSubcommand("resource") with ViashLogger { banner( "viash export resource", """Export an internal resource file""".stripMargin, @@ -528,7 +528,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val cli_schema = new DocumentedSubcommand("cli_schema") with ViashLogger { + object cli_schema extends DocumentedSubcommand("cli_schema") with ViashLogger { banner( "viash export cli_schema", """Export the schema of the Viash CLI as a JSON""".stripMargin, @@ -548,7 +548,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val cli_autocomplete = new DocumentedSubcommand("cli_autocomplete") with ViashLogger { + object cli_autocomplete extends DocumentedSubcommand("cli_autocomplete") with ViashLogger { banner( "viash export bash_autocomplete", """Export the autocomplete script as to be used in Bash or Zsh""".stripMargin, @@ -568,7 +568,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val config_schema = new DocumentedSubcommand("config_schema") with ViashLogger { + object config_schema extends DocumentedSubcommand("config_schema") with ViashLogger { banner( "viash export config_schema", """Export the schema of a Viash config as a JSON""".stripMargin, @@ -588,7 +588,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin ) } - val json_schema = new DocumentedSubcommand("json_schema") with ViashLogger { + object json_schema extends DocumentedSubcommand("json_schema") with ViashLogger { banner( "viash export json_schema", """Export the json schema to validate a Viash config""".stripMargin, @@ -633,7 +633,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin addSubcommand(test) addSubcommand(namespace) addSubcommand(config) - addSubcommand(export) + addSubcommand(`export`) shortSubcommandsHelp(true) diff --git a/src/main/scala/io/viash/cli/DocumentedSubcommand.scala b/src/main/scala/io/viash/cli/DocumentedSubcommand.scala index 70e4ca169..a1439678a 100644 --- a/src/main/scala/io/viash/cli/DocumentedSubcommand.scala +++ b/src/main/scala/io/viash/cli/DocumentedSubcommand.scala @@ -21,7 +21,7 @@ import org.rogach.scallop.Subcommand import org.rogach.scallop.ScallopOptionGroup import org.rogach.scallop.ValueConverter import org.rogach.scallop.ScallopOption -import scala.reflect.runtime.universe._ +// import scala.reflect.runtime.universe._ /** * Wrapper class for Subcommand to expose protected members @@ -86,9 +86,9 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co argName: String = "arg", hidden: Boolean = false, group: ScallopOptionGroup = null - )(implicit conv:ValueConverter[A], tag: TypeTag[A]): ScallopOption[A] = { + )(implicit conv:ValueConverter[A]/*, tag: TypeTag[A]*/): ScallopOption[A] = { - val `type` = tag.tpe + // val `type` = tag.tpe val cleanName = name match { case null => "" case _ => name @@ -103,7 +103,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co argName = Some(argName), hidden = hidden, choices = None, - `type` = `type`.toString(), + `type` = "TODO REWORK REFLECTION",//`type`.toString(), optType = "opt" ) registeredOpts = registeredOpts :+ registeredOpt @@ -151,9 +151,9 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co default: => Option[A] = None, hidden: Boolean = false, group: ScallopOptionGroup = null - )(implicit conv:ValueConverter[A], tag: TypeTag[A]) = { + )(implicit conv:ValueConverter[A]/*, tag: TypeTag[A]*/) = { - val `type` = tag.tpe + // val `type` = tag.tpe val cleanName = name match { case null => "" case _ => name @@ -168,7 +168,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co argName = None, hidden = hidden, choices = None, - `type` = `type`.toString, + `type` = "TODO REWORK REFLECTION",//`type`.toString, optType = "trailArgs" ) registeredOpts = registeredOpts :+ registeredOpt diff --git a/src/main/scala/io/viash/cli/package.scala b/src/main/scala/io/viash/cli/package.scala index 11dd143a4..1b2e4931b 100644 --- a/src/main/scala/io/viash/cli/package.scala +++ b/src/main/scala/io/viash/cli/package.scala @@ -18,7 +18,8 @@ package io.viash import io.circe.Encoder -import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +// import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoder import org.rogach.scallop.CliOption package object cli { diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index 8635a5da5..efb39ed74 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -47,6 +47,7 @@ import io.viash.lenses.ConfigLenses._ import Status._ import io.viash.wrapper.BashWrapper import scala.collection.immutable.ListMap +import io.viash.helpers.data_structures.oneOrMoreToList @description( """A Viash configuration is a YAML file which contains metadata to describe the behaviour and build target(s) of a component. @@ -642,7 +643,7 @@ object Config extends Logging { /* CONFIG Base: converted from json */ // convert Json into Config - val confBase = Convert.jsonToClass[Config](json2, uri.toString()) + val confBase = Config("")//Convert.jsonToClass[Config](json2, uri.toString()) /* CONFIG 0: apply values from package config */ // apply values from package config if need be @@ -668,11 +669,11 @@ object Config extends Logging { val conf1 = if (confMods.postparseCommands.nonEmpty) { // turn config back into json - val js = encodeConfig(conf0) + val js = Json.Null//encodeConfig(conf0) // apply config mods val modifiedJs = confMods(js, preparse = false) // turn json back into a config - Convert.jsonToClass[Config](modifiedJs, uri.toString()) + Config("")//Convert.jsonToClass[Config](modifiedJs, uri.toString()) } else { conf0 } @@ -683,8 +684,8 @@ object Config extends Logging { val resources = conf1.resources.map(_.copyWithAbsolutePath(parentURI, packageDir)) val tests = conf1.test_resources.map(_.copyWithAbsolutePath(parentURI, packageDir)) - val conf2a = resourcesLens.set(resources)(conf1) - val conf2 = testResourcesLens.set(tests)(conf2a) + val conf2a = resourcesLens.replace(resources)(conf1) + val conf2 = testResourcesLens.replace(tests)(conf2a) /* CONFIG 3: add info */ // gather git info @@ -826,5 +827,5 @@ object Config extends Logging { allConfigs } -val reservedParameters = List("-h", "--help", "--version", "---v", "---verbose", "---verbosity") + val reservedParameters = List("-h", "--help", "--version", "---v", "---verbose", "---verbosity") } diff --git a/src/main/scala/io/viash/config/ConfigMeta.scala b/src/main/scala/io/viash/config/ConfigMeta.scala index 63a56cedb..ffa71a2a3 100644 --- a/src/main/scala/io/viash/config/ConfigMeta.scala +++ b/src/main/scala/io/viash/config/ConfigMeta.scala @@ -62,7 +62,7 @@ object ConfigMeta { )) ) - val encodedConfig: Json = encodeConfig(anonymizedConfig) + val encodedConfig: Json = Json.Null//encodeConfig(anonymizedConfig) // drop empty & null values recursively except all "info" fields val cleanEncodedConfig = encodedConfig.dropEmptyRecursivelyExcept(Seq("info", ".engines.entrypoint", ".engines.cmd")) // get config.info and *do* clean it diff --git a/src/main/scala/io/viash/config/References.scala b/src/main/scala/io/viash/config/References.scala index c92bd7049..b24ee9d8d 100644 --- a/src/main/scala/io/viash/config/References.scala +++ b/src/main/scala/io/viash/config/References.scala @@ -18,7 +18,7 @@ package io.viash.config import io.viash.schemas._ -import io.viash.helpers.data_structures.OneOrMore +import io.viash.helpers.data_structures.{OneOrMore, listToOneOrMore} @description("A list of scholarly sources or publications relevant to the tools or analysis defined in the component. This is important for attribution, scientific reproducibility and transparency.") @example( diff --git a/src/main/scala/io/viash/config/arguments/package.scala b/src/main/scala/io/viash/config/arguments/package.scala index d16d932e0..52f30329e 100644 --- a/src/main/scala/io/viash/config/arguments/package.scala +++ b/src/main/scala/io/viash/config/arguments/package.scala @@ -18,11 +18,12 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen -import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ -import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ -import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ +// import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ +// import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ +// import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ +import io.circe.generic.semiauto.{deriveDecoder as deriveConfiguredDecoderFullChecks, deriveEncoder as deriveConfiguredEncoderStrict} import io.viash.exceptions.ConfigParserSubTypeException package object arguments { @@ -63,53 +64,54 @@ package object arguments { } // encoders and decoders for Argument - implicit val encodeStringArgument: Encoder.AsObject[StringArgument] = deriveConfiguredEncoderStrict[StringArgument] - implicit val encodeIntegerArgument: Encoder.AsObject[IntegerArgument] = deriveConfiguredEncoderStrict[IntegerArgument] - implicit val encodeLongArgument: Encoder.AsObject[LongArgument] = deriveConfiguredEncoderStrict[LongArgument] - implicit val encodeDoubleArgument: Encoder.AsObject[DoubleArgument] = deriveConfiguredEncoderStrict[DoubleArgument] - implicit val encodeBooleanArgumentR: Encoder.AsObject[BooleanArgument] = deriveConfiguredEncoderStrict[BooleanArgument] - implicit val encodeBooleanArgumentT: Encoder.AsObject[BooleanTrueArgument] = deriveConfiguredEncoderStrict[BooleanTrueArgument] - implicit val encodeBooleanArgumentF: Encoder.AsObject[BooleanFalseArgument] = deriveConfiguredEncoderStrict[BooleanFalseArgument] - implicit val encodeFileArgument: Encoder.AsObject[FileArgument] = deriveConfiguredEncoderStrict[FileArgument] + // implicit val encodeStringArgument: Encoder.AsObject[StringArgument] = deriveConfiguredEncoderStrict[StringArgument] + // implicit val encodeIntegerArgument: Encoder.AsObject[IntegerArgument] = deriveConfiguredEncoderStrict[IntegerArgument] + // implicit val encodeLongArgument: Encoder.AsObject[LongArgument] = deriveConfiguredEncoderStrict[LongArgument] + // implicit val encodeDoubleArgument: Encoder.AsObject[DoubleArgument] = deriveConfiguredEncoderStrict[DoubleArgument] + // implicit val encodeBooleanArgumentR: Encoder.AsObject[BooleanArgument] = deriveConfiguredEncoderStrict[BooleanArgument] + // implicit val encodeBooleanArgumentT: Encoder.AsObject[BooleanTrueArgument] = deriveConfiguredEncoderStrict[BooleanTrueArgument] + // implicit val encodeBooleanArgumentF: Encoder.AsObject[BooleanFalseArgument] = deriveConfiguredEncoderStrict[BooleanFalseArgument] + // implicit val encodeFileArgument: Encoder.AsObject[FileArgument] = deriveConfiguredEncoderStrict[FileArgument] implicit def encodeArgument[A <: Argument[_]]: Encoder[A] = Encoder.instance { par => val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - val objJson = par match { - case s: StringArgument => encodeStringArgument(s) - case s: IntegerArgument => encodeIntegerArgument(s) - case s: LongArgument => encodeLongArgument(s) - case s: DoubleArgument => encodeDoubleArgument(s) - case s: BooleanArgument => encodeBooleanArgumentR(s) - case s: BooleanTrueArgument => encodeBooleanArgumentT(s) - case s: BooleanFalseArgument => encodeBooleanArgumentF(s) - case s: FileArgument => encodeFileArgument(s) - } - objJson deepMerge typeJson + // val objJson = par match { + // case s: StringArgument => encodeStringArgument(s) + // case s: IntegerArgument => encodeIntegerArgument(s) + // case s: LongArgument => encodeLongArgument(s) + // case s: DoubleArgument => encodeDoubleArgument(s) + // case s: BooleanArgument => encodeBooleanArgumentR(s) + // case s: BooleanTrueArgument => encodeBooleanArgumentT(s) + // case s: BooleanFalseArgument => encodeBooleanArgumentF(s) + // case s: FileArgument => encodeFileArgument(s) + // } + // objJson deepMerge typeJson + typeJson } - implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeIntegerArgument: Decoder[IntegerArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeLongArgument: Decoder[LongArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeDoubleArgument: Decoder[DoubleArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeBooleanArgumentR: Decoder[BooleanArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeBooleanArgumentT: Decoder[BooleanTrueArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeBooleanArgumentF: Decoder[BooleanFalseArgument] = deriveConfiguredDecoderFullChecks - implicit val decodeFileArgument: Decoder[FileArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeIntegerArgument: Decoder[IntegerArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeLongArgument: Decoder[LongArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeDoubleArgument: Decoder[DoubleArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeBooleanArgumentR: Decoder[BooleanArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeBooleanArgumentT: Decoder[BooleanTrueArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeBooleanArgumentF: Decoder[BooleanFalseArgument] = deriveConfiguredDecoderFullChecks + // implicit val decodeFileArgument: Decoder[FileArgument] = deriveConfiguredDecoderFullChecks implicit def decodeDataArgument: Decoder[Argument[_]] = Decoder.instance { cursor => val decoder: Decoder[Argument[_]] = cursor.downField("type").as[String] match { - case Right("string") => decodeStringArgument.widen - case Right("integer") => decodeIntegerArgument.widen - case Right("long") => decodeLongArgument.widen - case Right("double") => decodeDoubleArgument.widen - case Right("boolean") => decodeBooleanArgumentR.widen - case Right("boolean_true") => decodeBooleanArgumentT.widen - case Right("boolean_false") => decodeBooleanArgumentF.widen - case Right("file") => decodeFileArgument.widen - case Right(typ) => invalidSubTypeDecoder[StringArgument](typ, List("string", "integer", "long", "double", "boolean", "boolean_true", "boolean_false", "file")).widen + // case Right("string") => decodeStringArgument.widen + // case Right("integer") => decodeIntegerArgument.widen + // case Right("long") => decodeLongArgument.widen + // case Right("double") => decodeDoubleArgument.widen + // case Right("boolean") => decodeBooleanArgumentR.widen + // case Right("boolean_true") => decodeBooleanArgumentT.widen + // case Right("boolean_false") => decodeBooleanArgumentF.widen + // case Right("file") => decodeFileArgument.widen + // case Right(typ) => invalidSubTypeDecoder[StringArgument](typ, List("string", "integer", "long", "double", "boolean", "boolean_true", "boolean_false", "file")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/config/dependencies/AbstractGitRepository.scala b/src/main/scala/io/viash/config/dependencies/AbstractGitRepository.scala index e5ac6c69c..79134ae4f 100644 --- a/src/main/scala/io/viash/config/dependencies/AbstractGitRepository.scala +++ b/src/main/scala/io/viash/config/dependencies/AbstractGitRepository.scala @@ -29,7 +29,7 @@ trait AbstractGitRepository extends Repository with Logging { val storePath: String @inline - protected def getLoggers(fn: String) = Seq[String => Unit] { str: String => debug(s"$fn: $str") } + protected def getLoggers(fn: String) = Seq[String => Unit] { (str: String) => debug(s"$fn: $str") } def copyRepo( `type`: String, diff --git a/src/main/scala/io/viash/config/dependencies/GithubRepositoryTrait.scala b/src/main/scala/io/viash/config/dependencies/GithubRepositoryTrait.scala index e513ad214..7f7b399bc 100644 --- a/src/main/scala/io/viash/config/dependencies/GithubRepositoryTrait.scala +++ b/src/main/scala/io/viash/config/dependencies/GithubRepositoryTrait.scala @@ -42,7 +42,7 @@ trait GithubRepositoryTrait extends AbstractGitRepository { def getCacheIdentifier(): Option[String] = Some(s"github-${repo.replace("/", "-")}${tag.map(_.prepended('-')).getOrElse("")}") - lazy val uri = s"https://github.com/$repo.git" + val uri = s"https://github.com/$repo.git" lazy val uri_ssh = s"git@github.com:$repo.git" val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian lazy val uri_nouser = s"https://${fakeCredentials}github.com/$repo.git" diff --git a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala index 6c65df069..eccfa0bd1 100644 --- a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala +++ b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala @@ -47,7 +47,7 @@ trait ViashhubRepositoryTrait extends AbstractGitRepository { def getCacheIdentifier(): Option[String] = Some(s"viashhub-${fullRepo.replace("/", "-")}${tag.map(_.prepended('-')).getOrElse("")}") - lazy val uri = s"https://viash-hub.com/$fullRepo.git" + val uri = s"https://viash-hub.com/$fullRepo.git" lazy val uri_ssh = s"git@viash-hub.com:$fullRepo.git" val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian lazy val uri_nouser = s"https://${fakeCredentials}viash-hub.com/$fullRepo.git" diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index e444735c3..e07755845 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -18,8 +18,8 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import cats.syntax.functor._ import dependencies.GithubRepository @@ -29,84 +29,84 @@ package object dependencies { import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // encoders and decoders for Argument - implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict - implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict - implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict - implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoderStrict - implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoderStrict + // implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict + // implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict + // implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict + // implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoderStrict + // implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoderStrict // Repositories _WithName are also of type Repository, so we must define an encoder for them as well - implicit def encodeRepository[A <: Repository]: Encoder[A] = Encoder.instance { - par => - val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - val objJson = par match { - case s: GitRepository => encodeGitRepository(s) - case s: GithubRepository => encodeGithubRepository(s) - case s: ViashhubRepository => encodeViashhubRepository(s) - case s: LocalRepository => encodeLocalRepository(s) - case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) - case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) - case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) - case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) - } - objJson deepMerge typeJson - } + // implicit def encodeRepository[A <: Repository]: Encoder[A] = Encoder.instance { + // par => + // val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + // val objJson = par match { + // case s: GitRepository => encodeGitRepository(s) + // case s: GithubRepository => encodeGithubRepository(s) + // case s: ViashhubRepository => encodeViashhubRepository(s) + // case s: LocalRepository => encodeLocalRepository(s) + // case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) + // case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) + // case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) + // case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) + // } + // objJson deepMerge typeJson + // } - implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoderStrict - implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoderStrict - implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoderStrict - implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoderStrict - implicit def encodeRepositoryWithName[A <: RepositoryWithName]: Encoder[A] = Encoder.instance { - par => - val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - val objJson = par match { - case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) - case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) - case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) - case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) - } - objJson deepMerge typeJson - } + // implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoderStrict + // implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoderStrict + // implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoderStrict + // implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoderStrict + // implicit def encodeRepositoryWithName[A <: RepositoryWithName]: Encoder[A] = Encoder.instance { + // par => + // val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + // val objJson = par match { + // case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) + // case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) + // case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) + // case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) + // } + // objJson deepMerge typeJson + // } - implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks - implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks - implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks - implicit val decodeViashhubRepository: Decoder[ViashhubRepository] = deriveConfiguredDecoderFullChecks - implicit val decodeLocalRepository: Decoder[LocalRepository] = deriveConfiguredDecoderFullChecks - implicit def decodeRepository: Decoder[Repository] = Decoder.instance { - cursor => - val decoder: Decoder[Repository] = - cursor.downField("type").as[String] match { - case Right("git") => decodeGitRepository.widen - case Right("github") => decodeGithubRepository.widen - case Right("vsh") => decodeViashhubRepository.widen - case Right("local") => decodeLocalRepository.widen - case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepository](typ, List("git", "github", "vsh", "local")).widen - case Left(exception) => throw exception - } + // implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks + // implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks + // implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks + // implicit val decodeViashhubRepository: Decoder[ViashhubRepository] = deriveConfiguredDecoderFullChecks + // implicit val decodeLocalRepository: Decoder[LocalRepository] = deriveConfiguredDecoderFullChecks + // implicit def decodeRepository: Decoder[Repository] = Decoder.instance { + // cursor => + // val decoder: Decoder[Repository] = + // cursor.downField("type").as[String] match { + // case Right("git") => decodeGitRepository.widen + // case Right("github") => decodeGithubRepository.widen + // case Right("vsh") => decodeViashhubRepository.widen + // case Right("local") => decodeLocalRepository.widen + // case Right(typ) => + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepository](typ, List("git", "github", "vsh", "local")).widen + // case Left(exception) => throw exception + // } - decoder(cursor) - } + // decoder(cursor) + // } - implicit val decodeGitRepositoryWithName: Decoder[GitRepositoryWithName] = deriveConfiguredDecoderFullChecks - implicit val decodeGithubRepositoryWithName: Decoder[GithubRepositoryWithName] = deriveConfiguredDecoderFullChecks - implicit val decodeViashhubRepositoryWithName: Decoder[ViashhubRepositoryWithName] = deriveConfiguredDecoderFullChecks - implicit val decodeLocalRepositoryWithName: Decoder[LocalRepositoryWithName] = deriveConfiguredDecoderFullChecks - implicit def decodeRepositoryWithName: Decoder[RepositoryWithName] = Decoder.instance { - cursor => - val decoder: Decoder[RepositoryWithName] = - cursor.downField("type").as[String] match { - case Right("git") => decodeGitRepositoryWithName.widen - case Right("github") => decodeGithubRepositoryWithName.widen - case Right("vsh") => decodeViashhubRepositoryWithName.widen - case Right("local") => decodeLocalRepositoryWithName.widen - case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepositoryWithName](typ, List("git", "github", "vsh", "local")).widen - case Left(exception) => throw exception - } + // implicit val decodeGitRepositoryWithName: Decoder[GitRepositoryWithName] = deriveConfiguredDecoderFullChecks + // implicit val decodeGithubRepositoryWithName: Decoder[GithubRepositoryWithName] = deriveConfiguredDecoderFullChecks + // implicit val decodeViashhubRepositoryWithName: Decoder[ViashhubRepositoryWithName] = deriveConfiguredDecoderFullChecks + // implicit val decodeLocalRepositoryWithName: Decoder[LocalRepositoryWithName] = deriveConfiguredDecoderFullChecks + // implicit def decodeRepositoryWithName: Decoder[RepositoryWithName] = Decoder.instance { + // cursor => + // val decoder: Decoder[RepositoryWithName] = + // cursor.downField("type").as[String] match { + // case Right("git") => decodeGitRepositoryWithName.widen + // case Right("github") => decodeGithubRepositoryWithName.widen + // case Right("vsh") => decodeViashhubRepositoryWithName.widen + // case Right("local") => decodeLocalRepositoryWithName.widen + // case Right(typ) => + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepositoryWithName](typ, List("git", "github", "vsh", "local")).widen + // case Left(exception) => throw exception + // } - decoder(cursor) - } + // decoder(cursor) + // } } diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index 14557cc39..bf119c06d 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -18,10 +18,14 @@ package io.viash import io.circe.{Decoder, Encoder, Json, HCursor, JsonObject} -import io.circe.generic.extras.Configuration -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -import io.viash.platforms.decodePlatform -import io.viash.functionality.decodeFunctionality +// import io.circe.generic.extras.Configuration +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoder +// import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoderStrict +// import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoderFullChecks +// import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoderWithValidationCheck +// import io.viash.platforms.decodePlatform +// import io.viash.functionality.decodeFunctionality import io.viash.exceptions.ConfigParserValidationException import config.ArgumentGroup @@ -34,14 +38,15 @@ import config.arguments._ package object config { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ + // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // encoders and decoders for Config - implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] - implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config].prepare{ + // implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] + // implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config] + /*.prepare{ checkDeprecation[Config](_) // map platforms to runners and engines .withFocus{ @@ -266,31 +271,32 @@ package object config { }, "Could not convert json to Config." ) + }*/ - implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder - implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks + // implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder + // implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks // encoder and decoder for Author - implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder - implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks + // implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder + // implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks // encoder and decoder for Requirements - implicit val encodeComputationalRequirements: Encoder.AsObject[ComputationalRequirements] = deriveConfiguredEncoder - implicit val decodeComputationalRequirements: Decoder[ComputationalRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeComputationalRequirements: Encoder.AsObject[ComputationalRequirements] = deriveConfiguredEncoder + // implicit val decodeComputationalRequirements: Decoder[ComputationalRequirements] = deriveConfiguredDecoderFullChecks // encoder and decoder for ArgumentGroup - implicit val encodeArgumentGroup: Encoder.AsObject[ArgumentGroup] = deriveConfiguredEncoder - implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks + // implicit val encodeArgumentGroup: Encoder.AsObject[ArgumentGroup] = deriveConfiguredEncoder + // implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks // encoder and decoder for Status, make string lowercase before decoding - implicit val encodeStatus: Encoder[Status] = Encoder.encodeEnumeration(Status) - implicit val decodeStatus: Decoder[Status] = Decoder.decodeEnumeration(Status).prepare { - _.withFocus(_.mapString(_.toLowerCase())) - } + // implicit val encodeStatus: Encoder[Status] = Encoder.encodeEnumeration(Status) + // implicit val decodeStatus: Decoder[Status] = Decoder.decodeEnumeration(Status).prepare { + // _.withFocus(_.mapString(_.toLowerCase())) + // } - implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict - implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks + // implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict + // implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks - implicit val encodeReferences: Encoder.AsObject[References] = deriveConfiguredEncoderStrict - implicit val decodeReferences: Decoder[References] = deriveConfiguredDecoderFullChecks + // implicit val encodeReferences: Encoder.AsObject[References] = deriveConfiguredEncoderStrict + // implicit val decodeReferences: Decoder[References] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/config/resources/PythonScript.scala b/src/main/scala/io/viash/config/resources/PythonScript.scala index 75349da6e..40bf1030c 100644 --- a/src/main/scala/io/viash/config/resources/PythonScript.scala +++ b/src/main/scala/io/viash/config/resources/PythonScript.scala @@ -47,39 +47,39 @@ case class PythonScript( def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = { val paramsCode = argsMetaAndDeps.map { case (dest, params) => val parSet = params.map { par => - // val env_name = par.VIASH_PAR - val env_name = Bash.getEscapedArgument(par.VIASH_PAR, "r'", "'", """\'""", """\'\"\'\"r\'""") + // val env_name = par.VIASH_PAR + val env_name = Bash.getEscapedArgument(par.VIASH_PAR, "r'", "'", """\'""", """\'\"\'\"r\'""") - val parse = par match { - case a: BooleanArgumentBase if a.multiple => - s"""list(map(lambda x: (x.lower() == 'true'), $env_name.split('${a.multiple_sep}')))""" - case a: IntegerArgument if a.multiple => - s"""list(map(int, $env_name.split('${a.multiple_sep}')))""" - case a: LongArgument if a.multiple => - s"""list(map(int, $env_name.split('${a.multiple_sep}')))""" - case a: DoubleArgument if a.multiple => - s"""list(map(float, $env_name.split('${a.multiple_sep}')))""" - case a: FileArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}')""" - case a: StringArgument if a.multiple => - s"""$env_name.split('${a.multiple_sep}')""" - case _: BooleanArgumentBase => s"""$env_name.lower() == 'true'""" - case _: IntegerArgument => s"""int($env_name)""" - case _: LongArgument => s"""int($env_name)""" - case _: DoubleArgument => s"""float($env_name)""" - case _: FileArgument => s"""$env_name""" - case _: StringArgument => s"""$env_name""" - } + val parse = par match { + case a: BooleanArgumentBase if a.multiple => + s"""list(map(lambda x: (x.lower() == 'true'), $env_name.split('${a.multiple_sep}')))""" + case a: IntegerArgument if a.multiple => + s"""list(map(int, $env_name.split('${a.multiple_sep}')))""" + case a: LongArgument if a.multiple => + s"""list(map(int, $env_name.split('${a.multiple_sep}')))""" + case a: DoubleArgument if a.multiple => + s"""list(map(float, $env_name.split('${a.multiple_sep}')))""" + case a: FileArgument if a.multiple => + s"""$env_name.split('${a.multiple_sep}')""" + case a: StringArgument if a.multiple => + s"""$env_name.split('${a.multiple_sep}')""" + case _: BooleanArgumentBase => s"""$env_name.lower() == 'true'""" + case _: IntegerArgument => s"""int($env_name)""" + case _: LongArgument => s"""int($env_name)""" + case _: DoubleArgument => s"""float($env_name)""" + case _: FileArgument => s"""$env_name""" + case _: StringArgument => s"""$env_name""" + } - val notFound = "None" + val notFound = "None" - s"""'${par.plainName}': $$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo $notFound; fi )""" - } + s"""'${par.plainName}': $$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo $notFound; fi )""" + } - s"""$dest = { - | ${parSet.mkString(",\n ")} - |} - |""".stripMargin + s"""$dest = { + | ${parSet.mkString(",\n ")} + |} + |""".stripMargin } ScriptInjectionMods(params = paramsCode.mkString) diff --git a/src/main/scala/io/viash/config/resources/ScalaScript.scala b/src/main/scala/io/viash/config/resources/ScalaScript.scala index d7be15eab..4bd7969f9 100644 --- a/src/main/scala/io/viash/config/resources/ScalaScript.scala +++ b/src/main/scala/io/viash/config/resources/ScalaScript.scala @@ -75,48 +75,48 @@ case class ScalaScript( // val env_name = par.VIASH_PAR val env_name = Bash.getEscapedArgument(par.VIASH_PAR, quo, """\"""", """\"\"\"+\"\\\"\"+\"\"\"""") - val parse = { par match { - case a: BooleanArgumentBase if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toLowerCase.toBoolean).toList""" - case a: IntegerArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toInt).toList""" - case a: LongArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toLong).toList""" - case a: DoubleArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toDouble).toList""" - case a: FileArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).toList""" - case a: StringArgument if a.multiple => - s"""$env_name.split($quo${a.multiple_sep}$quo).toList""" - case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => s"""Some($env_name.toLowerCase.toBoolean)""" - case a: IntegerArgument if !a.required => s"""Some($env_name.toInt)""" - case a: LongArgument if !a.required => s"""Some($env_name.toLong)""" - case a: DoubleArgument if !a.required => s"""Some($env_name.toDouble)""" - case a: FileArgument if !a.required => s"""Some($env_name)""" - case a: StringArgument if !a.required => s"""Some($env_name)""" - case _: BooleanArgumentBase => s"""$env_name.toLowerCase.toBoolean""" - case _: IntegerArgument => s"""$env_name.toInt""" - case _: LongArgument => s"""$env_name.toLong""" - case _: DoubleArgument => s"""$env_name.toDouble""" - case _: FileArgument => s"""$env_name""" - case _: StringArgument => s"""$env_name""" - }} - - // Todo: set as None if multiple is undefined - val notFound = par match { - case a: Argument[_] if a.multiple => Some("Nil") - case a: BooleanArgumentBase if a.flagValue.isDefined => None - case a: Argument[_] if !a.required => Some("None") - case _: Argument[_] => None - } + val parse = { par match { + case a: BooleanArgumentBase if a.multiple => + s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toLowerCase.toBoolean).toList""" + case a: IntegerArgument if a.multiple => + s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toInt).toList""" + case a: LongArgument if a.multiple => + s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toLong).toList""" + case a: DoubleArgument if a.multiple => + s"""$env_name.split($quo${a.multiple_sep}$quo).map(_.toDouble).toList""" + case a: FileArgument if a.multiple => + s"""$env_name.split($quo${a.multiple_sep}$quo).toList""" + case a: StringArgument if a.multiple => + s"""$env_name.split($quo${a.multiple_sep}$quo).toList""" + case a: BooleanArgumentBase if !a.required && a.flagValue.isEmpty => s"""Some($env_name.toLowerCase.toBoolean)""" + case a: IntegerArgument if !a.required => s"""Some($env_name.toInt)""" + case a: LongArgument if !a.required => s"""Some($env_name.toLong)""" + case a: DoubleArgument if !a.required => s"""Some($env_name.toDouble)""" + case a: FileArgument if !a.required => s"""Some($env_name)""" + case a: StringArgument if !a.required => s"""Some($env_name)""" + case _: BooleanArgumentBase => s"""$env_name.toLowerCase.toBoolean""" + case _: IntegerArgument => s"""$env_name.toInt""" + case _: LongArgument => s"""$env_name.toLong""" + case _: DoubleArgument => s"""$env_name.toDouble""" + case _: FileArgument => s"""$env_name""" + case _: StringArgument => s"""$env_name""" + }} + + // Todo: set as None if multiple is undefined + val notFound = par match { + case a: Argument[_] if a.multiple => Some("Nil") + case a: BooleanArgumentBase if a.flagValue.isDefined => None + case a: Argument[_] if !a.required => Some("None") + case _: Argument[_] => None + } - notFound match { - case Some(nf) => - s"""$$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo "$nf"; fi )""" - case None => - parse.replaceAll(quo, "\"\"\"") // undo quote escape as string is not part of echo + notFound match { + case Some(nf) => + s"""$$VIASH_DOLLAR$$( if [ ! -z $${${par.VIASH_PAR}+x} ]; then echo "$parse"; else echo "$nf"; fi )""" + case None => + parse.replaceAll(quo, "\"\"\"") // undo quote escape as string is not part of echo + } } - } s"""case class Viash${dest.capitalize}( | ${parClassTypes.mkString(",\n ")} diff --git a/src/main/scala/io/viash/config/resources/package.scala b/src/main/scala/io/viash/config/resources/package.scala index 2354dd5e0..030d61198 100644 --- a/src/main/scala/io/viash/config/resources/package.scala +++ b/src/main/scala/io/viash/config/resources/package.scala @@ -18,7 +18,7 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ import java.net.URI // for .widen @@ -26,89 +26,89 @@ import io.circe.ACursor package object resources { - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + // import io.viash.helpers.circe._ + // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ - implicit val encodeURI: Encoder[URI] = Encoder.instance { - uri => Json.fromString(uri.toString) - } - implicit val decodeURI: Decoder[URI] = Decoder.instance { - cursor => cursor.value.as[String].map(new URI(_)) - } + // implicit val encodeURI: Encoder[URI] = Encoder.instance { + // uri => Json.fromString(uri.toString) + // } + // implicit val decodeURI: Decoder[URI] = Decoder.instance { + // cursor => cursor.value.as[String].map(new URI(_)) + // } - // encoders and decoders for Object - implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoderStrict[BashScript] - implicit val encodePythonScript: Encoder.AsObject[PythonScript] = deriveConfiguredEncoderStrict[PythonScript] - implicit val encodeRScript: Encoder.AsObject[RScript] = deriveConfiguredEncoderStrict[RScript] - implicit val encodeJavaScriptScript: Encoder.AsObject[JavaScriptScript] = deriveConfiguredEncoderStrict[JavaScriptScript] - implicit val encodeNextflowScript: Encoder.AsObject[NextflowScript] = deriveConfiguredEncoderStrict[NextflowScript] - implicit val encodeScalaScript: Encoder.AsObject[ScalaScript] = deriveConfiguredEncoderStrict[ScalaScript] - implicit val encodeCSharpScript: Encoder.AsObject[CSharpScript] = deriveConfiguredEncoderStrict[CSharpScript] - implicit val encodeExecutable: Encoder.AsObject[Executable] = deriveConfiguredEncoderStrict[Executable] - implicit val encodePlainFile: Encoder.AsObject[PlainFile] = deriveConfiguredEncoderStrict[PlainFile] + // // encoders and decoders for Object + // implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoderStrict[BashScript] + // implicit val encodePythonScript: Encoder.AsObject[PythonScript] = deriveConfiguredEncoderStrict[PythonScript] + // implicit val encodeRScript: Encoder.AsObject[RScript] = deriveConfiguredEncoderStrict[RScript] + // implicit val encodeJavaScriptScript: Encoder.AsObject[JavaScriptScript] = deriveConfiguredEncoderStrict[JavaScriptScript] + // implicit val encodeNextflowScript: Encoder.AsObject[NextflowScript] = deriveConfiguredEncoderStrict[NextflowScript] + // implicit val encodeScalaScript: Encoder.AsObject[ScalaScript] = deriveConfiguredEncoderStrict[ScalaScript] + // implicit val encodeCSharpScript: Encoder.AsObject[CSharpScript] = deriveConfiguredEncoderStrict[CSharpScript] + // implicit val encodeExecutable: Encoder.AsObject[Executable] = deriveConfiguredEncoderStrict[Executable] + // implicit val encodePlainFile: Encoder.AsObject[PlainFile] = deriveConfiguredEncoderStrict[PlainFile] - implicit def encodeResource[A <: Resource]: Encoder[A] = Encoder.instance { - par => - val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - val objJson = par match { - case s: BashScript => encodeBashScript(s) - case s: PythonScript => encodePythonScript(s) - case s: RScript => encodeRScript(s) - case s: JavaScriptScript => encodeJavaScriptScript(s) - case s: NextflowScript => encodeNextflowScript(s) - case s: ScalaScript => encodeScalaScript(s) - case s: CSharpScript => encodeCSharpScript(s) - case s: Executable => encodeExecutable(s) - case s: PlainFile => encodePlainFile(s) - } - objJson deepMerge typeJson - } + // implicit def encodeResource[A <: Resource]: Encoder[A] = Encoder.instance { + // par => + // val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + // val objJson = par match { + // case s: BashScript => encodeBashScript(s) + // case s: PythonScript => encodePythonScript(s) + // case s: RScript => encodeRScript(s) + // case s: JavaScriptScript => encodeJavaScriptScript(s) + // case s: NextflowScript => encodeNextflowScript(s) + // case s: ScalaScript => encodeScalaScript(s) + // case s: CSharpScript => encodeCSharpScript(s) + // case s: Executable => encodeExecutable(s) + // case s: PlainFile => encodePlainFile(s) + // } + // objJson deepMerge typeJson + // } - val setDestToPathOrDefault = (default: String) => (aCursor: ACursor) => {aCursor.withFocus(js => { - js.mapObject{ obj => - // when json defines 'text' but no 'dest' set - // if has 'path' -> switch 'path' to 'dest' - // else if no 'path' or 'dest' -> set 'dest' to default value - if (obj.contains("text") && !obj.contains("dest")) { - if (obj.contains("path")) - obj.add("dest", obj("path").get).remove("path") - else - obj.add("dest", Json.fromString(default)) - } else { - obj - } - } - })} + // val setDestToPathOrDefault = (default: String) => (aCursor: ACursor) => {aCursor.withFocus(js => { + // js.mapObject{ obj => + // // when json defines 'text' but no 'dest' set + // // if has 'path' -> switch 'path' to 'dest' + // // else if no 'path' or 'dest' -> set 'dest' to default value + // if (obj.contains("text") && !obj.contains("dest")) { + // if (obj.contains("path")) + // obj.add("dest", obj("path").get).remove("path") + // else + // obj.add("dest", Json.fromString(default)) + // } else { + // obj + // } + // } + // })} - implicit val decodeBashScript: Decoder[BashScript] = deriveConfiguredDecoderFullChecks[BashScript].prepare { setDestToPathOrDefault("./script.sh") } - implicit val decodePythonScript: Decoder[PythonScript] = deriveConfiguredDecoderFullChecks[PythonScript].prepare { setDestToPathOrDefault("./script.py") } - implicit val decodeRScript: Decoder[RScript] = deriveConfiguredDecoderFullChecks[RScript].prepare { setDestToPathOrDefault("./script.R") } - implicit val decodeJavaScriptScript: Decoder[JavaScriptScript] = deriveConfiguredDecoderFullChecks[JavaScriptScript].prepare { setDestToPathOrDefault("./script.js") } - implicit val decodeNextflowScript: Decoder[NextflowScript] = deriveConfiguredDecoderFullChecks[NextflowScript].prepare { setDestToPathOrDefault("./script.nf") } - implicit val decodeScalaScript: Decoder[ScalaScript] = deriveConfiguredDecoderFullChecks[ScalaScript].prepare { setDestToPathOrDefault("./script.scala") } - implicit val decodeCSharpScript: Decoder[CSharpScript] = deriveConfiguredDecoderFullChecks[CSharpScript].prepare { setDestToPathOrDefault("./script.csx") } - implicit val decodeExecutable: Decoder[Executable] = deriveConfiguredDecoderFullChecks - implicit val decodePlainFile: Decoder[PlainFile] = deriveConfiguredDecoderFullChecks[PlainFile].prepare { setDestToPathOrDefault("./text.txt") } + // implicit val decodeBashScript: Decoder[BashScript] = deriveConfiguredDecoderFullChecks[BashScript].prepare { setDestToPathOrDefault("./script.sh") } + // implicit val decodePythonScript: Decoder[PythonScript] = deriveConfiguredDecoderFullChecks[PythonScript].prepare { setDestToPathOrDefault("./script.py") } + // implicit val decodeRScript: Decoder[RScript] = deriveConfiguredDecoderFullChecks[RScript].prepare { setDestToPathOrDefault("./script.R") } + // implicit val decodeJavaScriptScript: Decoder[JavaScriptScript] = deriveConfiguredDecoderFullChecks[JavaScriptScript].prepare { setDestToPathOrDefault("./script.js") } + // implicit val decodeNextflowScript: Decoder[NextflowScript] = deriveConfiguredDecoderFullChecks[NextflowScript].prepare { setDestToPathOrDefault("./script.nf") } + // implicit val decodeScalaScript: Decoder[ScalaScript] = deriveConfiguredDecoderFullChecks[ScalaScript].prepare { setDestToPathOrDefault("./script.scala") } + // implicit val decodeCSharpScript: Decoder[CSharpScript] = deriveConfiguredDecoderFullChecks[CSharpScript].prepare { setDestToPathOrDefault("./script.csx") } + // implicit val decodeExecutable: Decoder[Executable] = deriveConfiguredDecoderFullChecks + // implicit val decodePlainFile: Decoder[PlainFile] = deriveConfiguredDecoderFullChecks[PlainFile].prepare { setDestToPathOrDefault("./text.txt") } - implicit def decodeResource: Decoder[Resource] = Decoder.instance { - cursor => - val decoder: Decoder[Resource] = - cursor.downField("type").as[String] match { - case Right("bash_script") => decodeBashScript.widen - case Right("python_script") => decodePythonScript.widen - case Right("r_script") => decodeRScript.widen - case Right("javascript_script") => decodeJavaScriptScript.widen - case Right("nextflow_script") => decodeNextflowScript.widen - case Right("scala_script") => decodeScalaScript.widen - case Right("csharp_script") => decodeCSharpScript.widen - case Right("executable") => decodeExecutable.widen - case Right("file") => decodePlainFile.widen - case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, Script.companions.map(c => c.`type`) ++ List("executable", "file")).widen - case Left(_) => decodePlainFile.widen // default is a simple file - } + // implicit def decodeResource: Decoder[Resource] = Decoder.instance { + // cursor => + // val decoder: Decoder[Resource] = + // cursor.downField("type").as[String] match { + // case Right("bash_script") => decodeBashScript.widen + // case Right("python_script") => decodePythonScript.widen + // case Right("r_script") => decodeRScript.widen + // case Right("javascript_script") => decodeJavaScriptScript.widen + // case Right("nextflow_script") => decodeNextflowScript.widen + // case Right("scala_script") => decodeScalaScript.widen + // case Right("csharp_script") => decodeCSharpScript.widen + // case Right("executable") => decodeExecutable.widen + // case Right("file") => decodePlainFile.widen + // case Right(typ) => + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, Script.companions.map(c => c.`type`) ++ List("executable", "file")).widen + // case Left(_) => decodePlainFile.widen // default is a simple file + // } - decoder(cursor) - } + // decoder(cursor) + // } } \ No newline at end of file diff --git a/src/main/scala/io/viash/config_mods/ConfigModParser.scala b/src/main/scala/io/viash/config_mods/ConfigModParser.scala index e04f95cfe..221f64900 100644 --- a/src/main/scala/io/viash/config_mods/ConfigModParser.scala +++ b/src/main/scala/io/viash/config_mods/ConfigModParser.scala @@ -254,10 +254,10 @@ object ConfigModParser extends RegexParsers { def brackets: Parser[Condition] = "(" ~> condition <~ ")" def condition: Parser[Condition] = or def or: Parser[Condition] = rep1sep(and, "||") ^^ { - case comps => comps.reduceLeft(Or) + case comps => comps.reduceLeft(Or.apply) } def and: Parser[Condition] = rep1sep(comparison, "&&") ^^ { - case comps => comps.reduceLeft(And) + case comps => comps.reduceLeft(And.apply) } def comparison: Parser[Condition] = brackets | equals | notEquals | not | has | trueCond | falseCond def trueCond: Parser[Condition] = "true" ^^^ True diff --git a/src/main/scala/io/viash/config_mods/ConfigMods.scala b/src/main/scala/io/viash/config_mods/ConfigMods.scala index b5c70ab95..10bf5543e 100644 --- a/src/main/scala/io/viash/config_mods/ConfigMods.scala +++ b/src/main/scala/io/viash/config_mods/ConfigMods.scala @@ -19,7 +19,7 @@ package io.viash.config_mods // will need to be updated in a next version of scala // import scala.jdk.CollectionConverters._ -import scala.collection.JavaConverters._ +// import scala.collection.JavaConverters._ import java.nio.file.{Files, Path => JPath} import java.util.stream.Collectors diff --git a/src/main/scala/io/viash/engines/DockerEngine.scala b/src/main/scala/io/viash/engines/DockerEngine.scala index 1dd07e5f9..bf17cbe64 100644 --- a/src/main/scala/io/viash/engines/DockerEngine.scala +++ b/src/main/scala/io/viash/engines/DockerEngine.scala @@ -24,6 +24,7 @@ import io.viash.config.{Config, BuildInfo, Author} import io.viash.engines.requirements.{Requirements, DockerRequirements} import io.viash.helpers.{Escaper, Docker} import io.viash.wrapper.BashWrapper +import io.viash.helpers.data_structures.listToOneOrMore import io.viash.schemas._ import io.viash.helpers.DockerImageInfo diff --git a/src/main/scala/io/viash/engines/docker/package.scala b/src/main/scala/io/viash/engines/docker/package.scala index 7e0bacf38..ced79ff05 100644 --- a/src/main/scala/io/viash/engines/docker/package.scala +++ b/src/main/scala/io/viash/engines/docker/package.scala @@ -18,24 +18,24 @@ package io.viash.engines import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object docker { import io.viash.helpers.circe._ // encoder and decoder for resolvevolume - implicit val encodeResolveVolume: Encoder[DockerResolveVolume] = Encoder.instance { - v => Json.fromString(v.toString) - } - implicit val decodeResolveVolume: Decoder[DockerResolveVolume] = Decoder.instance { - cursor => - cursor.value.as[String].map(s => - s.toLowerCase() match { - case "manual" => Manual - case "auto" | "automatic" => Automatic - } - ) - } + // implicit val encodeResolveVolume: Encoder[DockerResolveVolume] = Encoder.instance { + // v => Json.fromString(v.toString) + // } + // implicit val decodeResolveVolume: Decoder[DockerResolveVolume] = Decoder.instance { + // cursor => + // cursor.value.as[String].map(s => + // s.toLowerCase() match { + // case "manual" => Manual + // case "auto" | "automatic" => Automatic + // } + // ) + // } } diff --git a/src/main/scala/io/viash/engines/package.scala b/src/main/scala/io/viash/engines/package.scala index e75fdca8c..1ad03d84e 100644 --- a/src/main/scala/io/viash/engines/package.scala +++ b/src/main/scala/io/viash/engines/package.scala @@ -18,7 +18,7 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen @@ -26,33 +26,33 @@ package object engines { import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - implicit val encodeDockerEngine: Encoder.AsObject[DockerEngine] = deriveConfiguredEncoder - implicit val decodeDockerEngine: Decoder[DockerEngine] = deriveConfiguredDecoderFullChecks - - implicit val encodeNativeEngine: Encoder.AsObject[NativeEngine] = deriveConfiguredEncoder - implicit val decodeNativeEngine: Decoder[NativeEngine] = deriveConfiguredDecoderFullChecks - - implicit def encodeEngine[A <: Engine]: Encoder[A] = Encoder.instance { - engine => - val typeJson = Json.obj("type" -> Json.fromString(engine.`type`)) - val objJson = engine match { - case s: DockerEngine => encodeDockerEngine(s) - case s: NativeEngine => encodeNativeEngine(s) - } - objJson deepMerge typeJson - } - - implicit def decodeEngine: Decoder[Engine] = Decoder.instance { - cursor => - val decoder: Decoder[Engine] = - cursor.downField("type").as[String] match { - case Right("docker") => decodeDockerEngine.widen - case Right("native") => decodeNativeEngine.widen - case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativeEngine](typ, List("docker", "native")).widen - case Left(exception) => throw exception - } - - decoder(cursor) - } + // implicit val encodeDockerEngine: Encoder.AsObject[DockerEngine] = deriveConfiguredEncoder + // implicit val decodeDockerEngine: Decoder[DockerEngine] = deriveConfiguredDecoderFullChecks + + // implicit val encodeNativeEngine: Encoder.AsObject[NativeEngine] = deriveConfiguredEncoder + // implicit val decodeNativeEngine: Decoder[NativeEngine] = deriveConfiguredDecoderFullChecks + + // implicit def encodeEngine[A <: Engine]: Encoder[A] = Encoder.instance { + // engine => + // val typeJson = Json.obj("type" -> Json.fromString(engine.`type`)) + // val objJson = engine match { + // case s: DockerEngine => encodeDockerEngine(s) + // case s: NativeEngine => encodeNativeEngine(s) + // } + // objJson deepMerge typeJson + // } + + // implicit def decodeEngine: Decoder[Engine] = Decoder.instance { + // cursor => + // val decoder: Decoder[Engine] = + // cursor.downField("type").as[String] match { + // case Right("docker") => decodeDockerEngine.widen + // case Right("native") => decodeNativeEngine.widen + // case Right(typ) => + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativeEngine](typ, List("docker", "native")).widen + // case Left(exception) => throw exception + // } + + // decoder(cursor) + // } } diff --git a/src/main/scala/io/viash/engines/requirements/package.scala b/src/main/scala/io/viash/engines/requirements/package.scala index da0d49cd3..686b1fe22 100644 --- a/src/main/scala/io/viash/engines/requirements/package.scala +++ b/src/main/scala/io/viash/engines/requirements/package.scala @@ -18,68 +18,69 @@ package io.viash.engines import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.Configuration -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.Configuration +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen package object requirements { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder - implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder + // implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodePythonRequirements: Encoder.AsObject[PythonRequirements] = deriveConfiguredEncoder - implicit val decodePythonRequirements: Decoder[PythonRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodePythonRequirements: Encoder.AsObject[PythonRequirements] = deriveConfiguredEncoder + // implicit val decodePythonRequirements: Decoder[PythonRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodeRubyRequirements: Encoder.AsObject[RubyRequirements] = deriveConfiguredEncoder - implicit val decodeRubyRequirements: Decoder[RubyRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeRubyRequirements: Encoder.AsObject[RubyRequirements] = deriveConfiguredEncoder + // implicit val decodeRubyRequirements: Decoder[RubyRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodeJavaScriptRequirements: Encoder.AsObject[JavaScriptRequirements] = deriveConfiguredEncoder - implicit val decodeJavaScriptRequirements: Decoder[JavaScriptRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeJavaScriptRequirements: Encoder.AsObject[JavaScriptRequirements] = deriveConfiguredEncoder + // implicit val decodeJavaScriptRequirements: Decoder[JavaScriptRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodeAptRequirements: Encoder.AsObject[AptRequirements] = deriveConfiguredEncoder - implicit val decodeAptRequirements: Decoder[AptRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeAptRequirements: Encoder.AsObject[AptRequirements] = deriveConfiguredEncoder + // implicit val decodeAptRequirements: Decoder[AptRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodeYumRequirements: Encoder.AsObject[YumRequirements] = deriveConfiguredEncoder - implicit val decodeYumRequirements: Decoder[YumRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeYumRequirements: Encoder.AsObject[YumRequirements] = deriveConfiguredEncoder + // implicit val decodeYumRequirements: Decoder[YumRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodeApkRequirements: Encoder.AsObject[ApkRequirements] = deriveConfiguredEncoder - implicit val decodeApkRequirements: Decoder[ApkRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeApkRequirements: Encoder.AsObject[ApkRequirements] = deriveConfiguredEncoder + // implicit val decodeApkRequirements: Decoder[ApkRequirements] = deriveConfiguredDecoderFullChecks - implicit val encodeDockerRequirements: Encoder.AsObject[DockerRequirements] = deriveConfiguredEncoder - implicit val decodeDockerRequirements: Decoder[DockerRequirements] = deriveConfiguredDecoderFullChecks + // implicit val encodeDockerRequirements: Encoder.AsObject[DockerRequirements] = deriveConfiguredEncoder + // implicit val decodeDockerRequirements: Decoder[DockerRequirements] = deriveConfiguredDecoderFullChecks implicit def encodeRequirements[A <: Requirements]: Encoder[A] = Encoder.instance { reqs => val typeJson = Json.obj("type" -> Json.fromString(reqs.`type`)) - val objJson = reqs match { - case s: ApkRequirements => encodeApkRequirements(s) - case s: AptRequirements => encodeAptRequirements(s) - case s: YumRequirements => encodeYumRequirements(s) - case s: DockerRequirements => encodeDockerRequirements(s) - case s: PythonRequirements => encodePythonRequirements(s) - case s: RRequirements => encodeRRequirements(s) - case s: JavaScriptRequirements => encodeJavaScriptRequirements(s) - case s: RubyRequirements => encodeRubyRequirements(s) - } - objJson deepMerge typeJson + // val objJson = reqs match { + // case s: ApkRequirements => encodeApkRequirements(s) + // case s: AptRequirements => encodeAptRequirements(s) + // case s: YumRequirements => encodeYumRequirements(s) + // case s: DockerRequirements => encodeDockerRequirements(s) + // case s: PythonRequirements => encodePythonRequirements(s) + // case s: RRequirements => encodeRRequirements(s) + // case s: JavaScriptRequirements => encodeJavaScriptRequirements(s) + // case s: RubyRequirements => encodeRubyRequirements(s) + // } + // objJson deepMerge typeJson + typeJson } implicit def decodeRequirements: Decoder[Requirements] = Decoder.instance { cursor => val decoder: Decoder[Requirements] = cursor.downField("type").as[String] match { - case Right("apk") => decodeApkRequirements.widen - case Right("apt") => decodeAptRequirements.widen - case Right("yum") => decodeYumRequirements.widen - case Right("docker") => decodeDockerRequirements.widen - case Right("python") => decodePythonRequirements.widen - case Right("r") => decodeRRequirements.widen - case Right("javascript") => decodeJavaScriptRequirements.widen - case Right("ruby") => decodeRubyRequirements.widen - case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ApkRequirements](typ, List("apk", "apt", "yum", "docker", "python", "r", "javascript", "ruby")).widen + // case Right("apk") => decodeApkRequirements.widen + // case Right("apt") => decodeAptRequirements.widen + // case Right("yum") => decodeYumRequirements.widen + // case Right("docker") => decodeDockerRequirements.widen + // case Right("python") => decodePythonRequirements.widen + // case Right("r") => decodeRRequirements.widen + // case Right("javascript") => decodeJavaScriptRequirements.widen + // case Right("ruby") => decodeRubyRequirements.widen + // case Right(typ) => + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ApkRequirements](typ, List("apk", "apt", "yum", "docker", "python", "r", "javascript", "ruby")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala index 50ab7916a..d455be4b0 100644 --- a/src/main/scala/io/viash/functionality/Functionality.scala +++ b/src/main/scala/io/viash/functionality/Functionality.scala @@ -19,7 +19,7 @@ package io.viash.functionality import io.circe.Json -import io.circe.generic.extras._ +// import io.circe.generic.extras._ import io.viash.config.arguments._ import io.viash.config.resources._ import io.viash.config.Status._ diff --git a/src/main/scala/io/viash/functionality/package.scala b/src/main/scala/io/viash/functionality/package.scala index 9901da81a..33d542e17 100644 --- a/src/main/scala/io/viash/functionality/package.scala +++ b/src/main/scala/io/viash/functionality/package.scala @@ -17,7 +17,7 @@ package io.viash -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.circe.{Decoder, Encoder, Json} import io.circe.ACursor @@ -36,13 +36,13 @@ import config.arguments._ package object functionality extends Logging { // import implicits import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ + // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // encoder and decoder for Functionality // implicit val encodeFunctionality: Encoder.AsObject[Functionality] = deriveConfiguredEncoderStrict[Functionality] - implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderFullChecks + // implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/helpers/Git.scala b/src/main/scala/io/viash/helpers/Git.scala index 743c15df5..a6b280648 100644 --- a/src/main/scala/io/viash/helpers/Git.scala +++ b/src/main/scala/io/viash/helpers/Git.scala @@ -28,7 +28,7 @@ case class GitInfo( object Git extends Logging { @inline - protected def getLoggers(fn: String) = Seq[String => Unit] { str: String => trace(s"$fn: $str") } + protected def getLoggers(fn: String) = Seq[String => Unit] { (str: String) => trace(s"$fn: $str") } def isGitRepo(path: File): Boolean = { val out = Exec.runCatch( diff --git a/src/main/scala/io/viash/helpers/Helper.scala b/src/main/scala/io/viash/helpers/Helper.scala index 92d46792c..5c443b0b6 100644 --- a/src/main/scala/io/viash/helpers/Helper.scala +++ b/src/main/scala/io/viash/helpers/Helper.scala @@ -21,6 +21,7 @@ import io.viash.config.Config import io.viash.Main import io.viash.config.ArgumentGroup import io.viash.config.arguments.{Argument, StringArgument, Output, IntegerArgument, LongArgument, DoubleArgument, FileArgument} +import io.viash.helpers.data_structures.oneOrMoreToList object Helper { private val maxWidth: Int = 80 diff --git a/src/main/scala/io/viash/helpers/IO.scala b/src/main/scala/io/viash/helpers/IO.scala index 39afb68f4..83677502a 100644 --- a/src/main/scala/io/viash/helpers/IO.scala +++ b/src/main/scala/io/viash/helpers/IO.scala @@ -20,7 +20,7 @@ package io.viash.helpers import java.io.{BufferedOutputStream, FileOutputStream, File, IOException} import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor, FileVisitOption} import java.nio.file.attribute.BasicFileAttributes -import scala.reflect.io.Directory +// import scala.reflect.io.Directory import java.net.URI import scala.io.{Codec, Source} import java.net.URL diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala index cab16076c..d4cfc7cb0 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -17,21 +17,21 @@ package io.viash.helpers.circe -import shapeless.Lazy -import scala.reflect.runtime.universe._ +// import shapeless.Lazy +// import scala.reflect.runtime.universe._ -import io.circe.Decoder -import io.circe.generic.extras.decoding.ConfiguredDecoder -import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +// import io.circe.Decoder +// import io.circe.generic.extras.decoding.ConfiguredDecoder +// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder object DeriveConfiguredDecoderFullChecks { - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - def deriveConfiguredDecoderFullChecks[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] - .validate( - validator[A], - s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." - ) - .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] ) + // def deriveConfiguredDecoderFullChecks[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + // .validate( + // validator[A], + // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." + // ) + // .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index f2ea76df4..1269c0841 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -17,12 +17,12 @@ package io.viash.helpers.circe -import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder import io.circe.{ Decoder, CursorOp } -import io.circe.generic.extras.decoding.ConfiguredDecoder +// import io.circe.generic.extras.decoding.ConfiguredDecoder -import scala.reflect.runtime.universe._ -import shapeless.Lazy +// import scala.reflect.runtime.universe._ +// import shapeless.Lazy import io.viash.schemas.ParameterSchema import io.circe.ACursor @@ -32,72 +32,72 @@ import io.viash.schemas.CollectedSchemas object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { - private def memberDeprecationCheck(name: String, history: List[CursorOp], parameters: List[ParameterSchema]): Unit = { - val schema = parameters.find(p => p.name == name).getOrElse(ParameterSchema("", "", "", None, None, None, None, None, None, None, None, false, false)) - - lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - - lazy val fullHistoryName = - if (historyString.isEmpty) { - s".$name" - } else { - s".$historyString.$name" - } - - schema.deprecated match { - case Some(d) => - info(s"Warning: $fullHistoryName is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") - case _ => - } - schema.removed match { - case Some(r) => - info(s"Error: $fullHistoryName was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") - case _ => - } - if (schema.hasInternalFunctionality) { - error(s"Error: $fullHistoryName is internal functionality.") - throw new RuntimeException(s"Internal functionality used: $fullHistoryName") - } - } - - private def selfDeprecationCheck(parameters: List[ParameterSchema]): Unit = { - val schema = parameters.find(p => p.name == "__this__").get - - schema.deprecated match { - case Some(d) => - info(s"Warning: ${schema.`type`} is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") - case _ => - } - schema.removed match { - case Some(r) => - info(s"Error: ${schema.`type`} was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") - case _ => - } - } - - // - def checkDeprecation[A](cursor: ACursor)(implicit tag: TypeTag[A]) : ACursor = { - val parameters = CollectedSchemas.getParameters[A]() - - selfDeprecationCheck(parameters) - - // check each defined 'key' value - for (key <- cursor.keys.getOrElse(Nil)) { - val isEmpty = - cursor.downField(key).focus.get match { - case value if value.isNull => true - case value if value.isArray => value.asArray.get.isEmpty - case value if value.isObject => value.asObject.get.isEmpty - case _ => false - } - if (!isEmpty) { - memberDeprecationCheck(key, cursor.history, parameters) - } - } - cursor // return unchanged json info - } - - // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here - def deriveConfiguredDecoderWithDeprecationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] - .prepare( checkDeprecation[A] ) + // private def memberDeprecationCheck(name: String, history: List[CursorOp], parameters: List[ParameterSchema]): Unit = { + // val schema = parameters.find(p => p.name == name).getOrElse(ParameterSchema("", "", "", None, None, None, None, None, None, None, None, false, false)) + + // lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") + + // lazy val fullHistoryName = + // if (historyString.isEmpty) { + // s".$name" + // } else { + // s".$historyString.$name" + // } + + // schema.deprecated match { + // case Some(d) => + // info(s"Warning: $fullHistoryName is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + // case _ => + // } + // schema.removed match { + // case Some(r) => + // info(s"Error: $fullHistoryName was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + // case _ => + // } + // if (schema.hasInternalFunctionality) { + // error(s"Error: $fullHistoryName is internal functionality.") + // throw new RuntimeException(s"Internal functionality used: $fullHistoryName") + // } + // } + + // private def selfDeprecationCheck(parameters: List[ParameterSchema]): Unit = { + // val schema = parameters.find(p => p.name == "__this__").get + + // schema.deprecated match { + // case Some(d) => + // info(s"Warning: ${schema.`type`} is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + // case _ => + // } + // schema.removed match { + // case Some(r) => + // info(s"Error: ${schema.`type`} was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + // case _ => + // } + // } + + // // + // def checkDeprecation[A](cursor: ACursor)(implicit tag: TypeTag[A]) : ACursor = { + // val parameters = CollectedSchemas.getParameters[A]() + + // selfDeprecationCheck(parameters) + + // // check each defined 'key' value + // for (key <- cursor.keys.getOrElse(Nil)) { + // val isEmpty = + // cursor.downField(key).focus.get match { + // case value if value.isNull => true + // case value if value.isArray => value.asArray.get.isEmpty + // case value if value.isObject => value.asObject.get.isEmpty + // case _ => false + // } + // if (!isEmpty) { + // memberDeprecationCheck(key, cursor.history, parameters) + // } + // } + // cursor // return unchanged json info + // } + + // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here + // def deriveConfiguredDecoderWithDeprecationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + // .prepare( checkDeprecation[A] ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 6f08e3925..5e62c2d0f 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -17,12 +17,12 @@ package io.viash.helpers.circe -import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder import io.circe.{ Decoder, CursorOp } -import io.circe.generic.extras.decoding.ConfiguredDecoder +// import io.circe.generic.extras.decoding.ConfiguredDecoder -import scala.reflect.runtime.universe._ -import shapeless.Lazy +// import scala.reflect.runtime.universe._ +// import shapeless.Lazy import io.viash.schemas.ParameterSchema import io.circe.ACursor @@ -34,51 +34,51 @@ object DeriveConfiguredDecoderWithValidationCheck { // Validate the json can correctly converted to the required type by actually converting it. // Throw an exception when the conversion fails. - def validator[A](pred: HCursor)(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Boolean = { - val d = deriveConfiguredDecoder[A] - val v = d(pred) + // def validator[A](pred: HCursor)(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Boolean = { + // val d = deriveConfiguredDecoder[A] + // val v = d(pred) - v.fold(error => { - val usedFields = pred.value.asObject.map(_.keys.toSeq) - val validFields = typeOf[A].members.filter(m => !m.isMethod).map(_.name.toString.strip()).toSeq - val invalidFields = usedFields.map(_.diff(validFields)) + // v.fold(error => { + // val usedFields = pred.value.asObject.map(_.keys.toSeq) + // val validFields = typeOf[A].members.filter(m => !m.isMethod).map(_.name.toString.strip()).toSeq + // val invalidFields = usedFields.map(_.diff(validFields)) - val fieldsHint = invalidFields match { - case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}") - case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}") - case _ => None - } + // val fieldsHint = invalidFields match { + // case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}") + // case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}") + // case _ => None + // } - val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") + // val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - val hint = (fieldsHint, historyString, error.message) match { - case (Some(a), h, _) if h != "" => Some(s".$h -> $a") - case (Some(a), _, _) => Some(a) - case (None, h, m) if h != "" => Some(s".$h -> $m") - case _ => None - } + // val hint = (fieldsHint, historyString, error.message) match { + // case (Some(a), h, _) if h != "" => Some(s".$h -> $a") + // case (Some(a), _, _) => Some(a) + // case (None, h, m) if h != "" => Some(s".$h -> $m") + // case _ => None + // } - throw new ConfigParserValidationException(typeOf[A].baseClasses.head.fullName, pred.value.toString(), hint) - false - }, _ => true) - } + // throw new ConfigParserValidationException(typeOf[A].baseClasses.head.fullName, pred.value.toString(), hint) + // false + // }, _ => true) + // } // Attempts to convert the json to the desired class. Throw an exception if the conversion fails. - def deriveConfiguredDecoderWithValidationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] - .validate( - validator[A], - s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." - ) + // def deriveConfiguredDecoderWithValidationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + // .validate( + // validator[A], + // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." + // ) - // Dummy decoder to generate exceptions when an invalid type is specified - // We need a valid class type to be specified - def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] - .validate( - pred => { - throw new ConfigParserSubTypeException(tpe, validTypes, pred.value.toString()) - false - }, - s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." - ) + // // Dummy decoder to generate exceptions when an invalid type is specified + // // We need a valid class type to be specified + // def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + // .validate( + // pred => { + // throw new ConfigParserSubTypeException(tpe, validTypes, pred.value.toString()) + // false + // }, + // s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." + // ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala index 78e78f585..47fd85888 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala @@ -19,24 +19,24 @@ package io.viash.helpers.circe import io.circe.{Encoder, Json, HCursor} // import io.circe.generic.extras.Configuration -import io.circe.generic.extras.semiauto.deriveConfiguredEncoder -import io.circe.generic.extras.encoding.ConfiguredAsObjectEncoder +// import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +// import io.circe.generic.extras.encoding.ConfiguredAsObjectEncoder -import scala.reflect.runtime.universe._ -import shapeless.Lazy +// import scala.reflect.runtime.universe._ +// import shapeless.Lazy import io.viash.schemas.ParameterSchema import io.viash.schemas.CollectedSchemas object DeriveConfiguredEncoderStrict { - final def deriveConfiguredEncoderStrict[T](implicit encode: Lazy[ConfiguredAsObjectEncoder[T]], tag: TypeTag[T]) = deriveConfiguredEncoder[T] - .mapJsonObject{ jsonObject => - val parameters = CollectedSchemas.getParameters[T]() - jsonObject.filterKeys( k => - parameters - .find(_.name == k) // find the correct parameter - .map(!_.hasInternalFunctionality) // check if it has the 'internalFunctionality' annotation - .getOrElse(true) // fallback, shouldn't really happen - ) - } + // final def deriveConfiguredEncoderStrict[T](implicit encode: Lazy[ConfiguredAsObjectEncoder[T]], tag: TypeTag[T]) = deriveConfiguredEncoder[T] + // .mapJsonObject{ jsonObject => + // val parameters = CollectedSchemas.getParameters[T]() + // jsonObject.filterKeys( k => + // parameters + // .find(_.name == k) // find the correct parameter + // .map(!_.hasInternalFunctionality) // check if it has the 'internalFunctionality' annotation + // .getOrElse(true) // fallback, shouldn't really happen + // ) + // } } diff --git a/src/main/scala/io/viash/helpers/circe/RichJson.scala b/src/main/scala/io/viash/helpers/circe/RichJson.scala index 9ae2cc989..72ba31952 100644 --- a/src/main/scala/io/viash/helpers/circe/RichJson.scala +++ b/src/main/scala/io/viash/helpers/circe/RichJson.scala @@ -21,7 +21,7 @@ import java.net.URI import io.circe.Json import io.circe.JsonObject -import io.circe.generic.extras.Configuration +// import io.circe.generic.extras.Configuration import io.circe.{Json, Printer => JsonPrinter} import io.circe.yaml.{Printer => YamlPrinter} diff --git a/src/main/scala/io/viash/helpers/circe/package.scala b/src/main/scala/io/viash/helpers/circe/package.scala index f3b40edad..19288a91b 100644 --- a/src/main/scala/io/viash/helpers/circe/package.scala +++ b/src/main/scala/io/viash/helpers/circe/package.scala @@ -18,14 +18,14 @@ package io.viash.helpers import io.circe._ -import io.circe.generic.extras.Configuration +// import io.circe.generic.extras.Configuration import java.net.URI import data_structures.OneOrMore import java.nio.file.Paths package object circe { - implicit val customConfig: Configuration = - Configuration.default.withDefaults.withStrictDecoding + // implicit val customConfig: Configuration = + // Configuration.default.withDefaults.withStrictDecoding // encoder and decoder for Either implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { @@ -38,7 +38,7 @@ package object circe { // encoder and decoder for OneOrMore implicit def encodeOneOrMore[A](implicit enc: Encoder[List[A]]): Encoder[OneOrMore[A]] = { - oom: OneOrMore[A] => if (oom == null) enc(Nil) else enc(oom.toList) + (oom: OneOrMore[A]) => if (oom == null) enc(Nil) else enc(oom.toList) } implicit def decodeOneOrMore[A](implicit da: Decoder[A], dl: Decoder[List[A]]): Decoder[OneOrMore[A]] = { diff --git a/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala b/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala index 06511084a..b537318c4 100644 --- a/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala +++ b/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala @@ -28,13 +28,13 @@ object AppliedConfigLenses { val appliedEnginesLens = GenLens[AppliedConfig](_.engines) val appliedRunnerLens = GenLens[AppliedConfig](_.runner) - val enginesLens = configLens ^|-> ConfigLenses.enginesLens - val runnersLens = configLens ^|-> ConfigLenses.runnersLens + val enginesLens = configLens andThen ConfigLenses.enginesLens + val runnersLens = configLens andThen ConfigLenses.runnersLens - val configNameLens = configLens ^|-> ConfigLenses.nameLens - val configVersionLens = configLens ^|-> ConfigLenses.versionLens - val configRequirementsLens = configLens ^|-> ConfigLenses.requirementsLens - val configDependenciesLens = configLens ^|-> ConfigLenses.dependenciesLens - val configRepositoriesLens = configLens ^|-> ConfigLenses.repositoriesLens - val configResourcesLens = configLens ^|-> ConfigLenses.resourcesLens + val configNameLens = configLens andThen ConfigLenses.nameLens + val configVersionLens = configLens andThen ConfigLenses.versionLens + val configRequirementsLens = configLens andThen ConfigLenses.requirementsLens + val configDependenciesLens = configLens andThen ConfigLenses.dependenciesLens + val configRepositoriesLens = configLens andThen ConfigLenses.repositoriesLens + val configResourcesLens = configLens andThen ConfigLenses.resourcesLens } diff --git a/src/main/scala/io/viash/lenses/ConfigLenses.scala b/src/main/scala/io/viash/lenses/ConfigLenses.scala index 02358d306..ed9c4422a 100644 --- a/src/main/scala/io/viash/lenses/ConfigLenses.scala +++ b/src/main/scala/io/viash/lenses/ConfigLenses.scala @@ -39,6 +39,6 @@ object ConfigLenses { val licenseLens = GenLens[Config](_.license) val linksLens = GenLens[Config](_.links) - val linksRepositoryLens = linksLens ^|-> repositoryLens - val linksDockerRegistryLens = linksLens ^|-> LinksLenses.dockerRegistryLens + val linksRepositoryLens = linksLens andThen repositoryLens + val linksDockerRegistryLens = linksLens andThen LinksLenses.dockerRegistryLens } diff --git a/src/main/scala/io/viash/packageConfig/PackageConfig.scala b/src/main/scala/io/viash/packageConfig/PackageConfig.scala index 5de9dc6cc..fa216a7f0 100644 --- a/src/main/scala/io/viash/packageConfig/PackageConfig.scala +++ b/src/main/scala/io/viash/packageConfig/PackageConfig.scala @@ -21,6 +21,7 @@ import java.nio.file.{Files, Path, Paths} import io.viash.schemas._ import io.viash.helpers.data_structures.OneOrMore +import io.viash.helpers.data_structures.listToOneOrMore import io.viash.helpers.IO import io.viash.helpers.circe._ import io.circe.Json @@ -245,7 +246,7 @@ object PackageConfig { /* PACKAGE 0: converted from json */ // convert Json into ViashPackage - val pack0 = Convert.jsonToClass[PackageConfig](json, path.toString()) + val pack0 = PackageConfig()//Convert.jsonToClass[PackageConfig](json, path.toString()) /* PACKAGE 1: make resources absolute */ // make paths absolute diff --git a/src/main/scala/io/viash/packageConfig/package.scala b/src/main/scala/io/viash/packageConfig/package.scala index f080e814e..6654d2fe4 100644 --- a/src/main/scala/io/viash/packageConfig/package.scala +++ b/src/main/scala/io/viash/packageConfig/package.scala @@ -21,9 +21,9 @@ import io.circe.{Decoder, Encoder} package object packageConfig { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ - implicit val encodePackageConfig: Encoder.AsObject[PackageConfig] = deriveConfiguredEncoderStrict - implicit val decodePackageConfig: Decoder[PackageConfig] = deriveConfiguredDecoderFullChecks + // implicit val encodePackageConfig: Encoder.AsObject[PackageConfig] = deriveConfiguredEncoderStrict + // implicit val decodePackageConfig: Decoder[PackageConfig] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/platforms/package.scala b/src/main/scala/io/viash/platforms/package.scala index 14dfc3f78..5944be862 100644 --- a/src/main/scala/io/viash/platforms/package.scala +++ b/src/main/scala/io/viash/platforms/package.scala @@ -18,7 +18,8 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoder import cats.syntax.functor._ // for .widen package object platforms { @@ -26,13 +27,13 @@ package object platforms { import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ // implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder - implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks + // implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks // implicit val encodeNextflowPlatform: Encoder.AsObject[NextflowPlatform] = deriveConfiguredEncoder - implicit val decodeNextflowPlatform: Decoder[NextflowPlatform] = deriveConfiguredDecoderFullChecks + // implicit val decodeNextflowPlatform: Decoder[NextflowPlatform] = deriveConfiguredDecoderFullChecks // implicit val encodeNativePlatform: Encoder.AsObject[NativePlatform] = deriveConfiguredEncoder - implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderFullChecks + // implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderFullChecks // implicit def encodePlatform[A <: Platform]: Encoder[A] = Encoder.instance { // platform => @@ -45,19 +46,19 @@ package object platforms { // objJson deepMerge typeJson // } - implicit def decodePlatform: Decoder[Platform] = Decoder.instance { - cursor => - val decoder: Decoder[Platform] = - cursor.downField("type").as[String] match { - case Right("docker") => decodeDockerPlatform.widen - case Right("native") => decodeNativePlatform.widen - case Right("nextflow") => decodeNextflowPlatform.widen - case Right(typ) => - //throw new RuntimeException("Type " + typ + " is not recognised.") - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativePlatform](typ, List("docker", "native", "nextflow")).widen - case Left(exception) => throw exception - } - - decoder(cursor) - } + // implicit def decodePlatform: Decoder[Platform] = Decoder.instance { + // cursor => + // val decoder: Decoder[Platform] = + // cursor.downField("type").as[String] match { + // case Right("docker") => decodeDockerPlatform.widen + // case Right("native") => decodeNativePlatform.widen + // case Right("nextflow") => decodeNextflowPlatform.widen + // case Right(typ) => + // //throw new RuntimeException("Type " + typ + " is not recognised.") + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativePlatform](typ, List("docker", "native", "nextflow")).widen + // case Left(exception) => throw exception + // } + + // decoder(cursor) + // } } diff --git a/src/main/scala/io/viash/runners/NextflowRunner.scala b/src/main/scala/io/viash/runners/NextflowRunner.scala index 2fabecd8e..90df9416b 100644 --- a/src/main/scala/io/viash/runners/NextflowRunner.scala +++ b/src/main/scala/io/viash/runners/NextflowRunner.scala @@ -140,7 +140,7 @@ final case class NextflowRunner( case Some(_) => throw new RuntimeException(s"NextflowRunner 'container' variable: Engine $container is not a Docker Engine") case None => None - case _ => ??? + case null => ??? } } diff --git a/src/main/scala/io/viash/runners/executable/package.scala b/src/main/scala/io/viash/runners/executable/package.scala index a5d161b06..8ee1de674 100644 --- a/src/main/scala/io/viash/runners/executable/package.scala +++ b/src/main/scala/io/viash/runners/executable/package.scala @@ -18,24 +18,24 @@ package io.viash.runners import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object executable { import io.viash.helpers.circe._ // encoder and decoder for setup strategy - implicit val encodeSetupStrategy: Encoder[DockerSetupStrategy] = Encoder.instance { - dss => Json.fromString(dss.id.toLowerCase) - } - implicit val decodeSetupStrategy: Decoder[DockerSetupStrategy] = Decoder.instance { - cursor => - cursor.value.as[String].map { s => - val id = s.toLowerCase.replaceAll("_", "") - DockerSetupStrategy.map.applyOrElse(id, - (key: String) => throw new Exception(s"Docker Setup Strategy '$key' not found.") - ) - } - } + // implicit val encodeSetupStrategy: Encoder[DockerSetupStrategy] = Encoder.instance { + // dss => Json.fromString(dss.id.toLowerCase) + // } + // implicit val decodeSetupStrategy: Decoder[DockerSetupStrategy] = Decoder.instance { + // cursor => + // cursor.value.as[String].map { s => + // val id = s.toLowerCase.replaceAll("_", "") + // DockerSetupStrategy.map.applyOrElse(id, + // (key: String) => throw new Exception(s"Docker Setup Strategy '$key' not found.") + // ) + // } + // } } diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala index ab23fa1c8..e80ae101d 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala @@ -20,6 +20,7 @@ package io.viash.runners.nextflow import scala.collection.immutable.ListMap import io.viash.schemas._ import io.viash.helpers.data_structures.OneOrMore +import io.viash.helpers.data_structures.listToOneOrMore @description("Allows tweaking how the Nextflow Config file is generated.") @since("Viash 0.7.4") diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala index c2f3c096c..ffb1343de 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala @@ -44,7 +44,7 @@ object NextflowHelper { lazy val dataflowHelper: String = readSource("DataflowHelper.nf") def generateConfigStr(config: Config): String = { - val configJson = config.asJson.dropEmptyRecursively + val configJson = Json.Null//config.asJson.dropEmptyRecursively val configJsonStr = configJson .toFormattedString("json") .replace("\\\\", "\\\\\\\\") @@ -89,10 +89,10 @@ object NextflowHelper { def generateDefaultWorkflowArgs(config: Config, directives: NextflowDirectives, auto: NextflowAuto, debug: Boolean): String = { // override container val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) - val dirJson = directives.asJson.dropEmptyRecursively + val dirJson = Json.Null//directives.asJson.dropEmptyRecursively val dirJson2 = if (dirJson.isNull) Json.obj() else dirJson - val autoJson = auto.asJson.dropEmptyRecursively + val autoJson = Json.Null//auto.asJson.dropEmptyRecursively s"""[ | // key to be used to trace the process and determine output names diff --git a/src/main/scala/io/viash/runners/nextflow/package.scala b/src/main/scala/io/viash/runners/nextflow/package.scala index 6dd0a7ff1..a11b455ed 100644 --- a/src/main/scala/io/viash/runners/nextflow/package.scala +++ b/src/main/scala/io/viash/runners/nextflow/package.scala @@ -18,18 +18,20 @@ package io.viash.runners import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoder +import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoderFullChecks package object nextflow { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder - implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks + // implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder + // implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks - implicit val encodeNextflowAuto: Encoder.AsObject[NextflowAuto] = deriveConfiguredEncoder - implicit val decodeNextflowAuto: Decoder[NextflowAuto] = deriveConfiguredDecoderFullChecks + // implicit val encodeNextflowAuto: Encoder.AsObject[NextflowAuto] = deriveConfiguredEncoder + // implicit val decodeNextflowAuto: Decoder[NextflowAuto] = deriveConfiguredDecoderFullChecks - implicit val encodeNextflowConfig: Encoder.AsObject[NextflowConfig] = deriveConfiguredEncoder - implicit val decodeNextflowConfig: Decoder[NextflowConfig] = deriveConfiguredDecoderFullChecks + // implicit val encodeNextflowConfig: Encoder.AsObject[NextflowConfig] = deriveConfiguredEncoder + // implicit val decodeNextflowConfig: Decoder[NextflowConfig] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/runners/package.scala b/src/main/scala/io/viash/runners/package.scala index ce8220eee..8f64f12c1 100644 --- a/src/main/scala/io/viash/runners/package.scala +++ b/src/main/scala/io/viash/runners/package.scala @@ -18,40 +18,41 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import cats.syntax.functor._ // for .widen package object runners { import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = deriveConfiguredEncoder - implicit val decodeExecutableRunner: Decoder[ExecutableRunner] = deriveConfiguredDecoderFullChecks - - implicit val encodeNextflowRunner: Encoder.AsObject[NextflowRunner] = deriveConfiguredEncoder - implicit val decodeNextflowRunner: Decoder[NextflowRunner] = deriveConfiguredDecoderFullChecks - - implicit def encodeRunner[A <: Runner]: Encoder[A] = Encoder.instance { - runner => - val typeJson = Json.obj("type" -> Json.fromString(runner.`type`)) - val objJson = runner match { - case s: ExecutableRunner => encodeExecutableRunner(s) - case s: NextflowRunner => encodeNextflowRunner(s) - } - objJson deepMerge typeJson - } - - implicit def decodeRunner: Decoder[Runner] = Decoder.instance { - cursor => - val decoder: Decoder[Runner] = - cursor.downField("type").as[String] match { - case Right("executable") => decodeExecutableRunner.widen - case Right("nextflow") => decodeNextflowRunner.widen - case Right(typ) => - DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ExecutableRunner](typ, List("executable", "nextflow")).widen - case Left(exception) => throw exception - } - - decoder(cursor) - } + // implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = deriveEncoder + // implicit val decodeExecutableRunner: Decoder[ExecutableRunner] = deriveConfiguredDecoderFullChecks + + // implicit val encodeNextflowRunner: Encoder.AsObject[NextflowRunner] = deriveEncoder + // implicit val decodeNextflowRunner: Decoder[NextflowRunner] = deriveConfiguredDecoderFullChecks + + // implicit def encodeRunner[A <: Runner]: Encoder[A] = Encoder.instance { + // runner => + // val typeJson = Json.obj("type" -> Json.fromString(runner.`type`)) + // val objJson = runner match { + // case s: ExecutableRunner => encodeExecutableRunner(s) + // case s: NextflowRunner => encodeNextflowRunner(s) + // } + // objJson deepMerge typeJson + // } + + // implicit def decodeRunner: Decoder[Runner] = Decoder.instance { + // cursor => + // val decoder: Decoder[Runner] = + // cursor.downField("type").as[String] match { + // case Right("executable") => decodeExecutableRunner.widen + // case Right("nextflow") => decodeNextflowRunner.widen + // case Right(typ) => + // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ExecutableRunner](typ, List("executable", "nextflow")).widen + // case Left(exception) => throw exception + // } + + // decoder(cursor) + // } } diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 35f98614f..869916c5f 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -17,10 +17,12 @@ package io.viash.schemas -import scala.reflect.runtime.universe._ +// import scala.reflect.runtime.universe._ import io.circe.{Encoder, Printer => JsonPrinter} import io.circe.syntax.EncoderOps -import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +// import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoder +import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoderStrict import io.viash.functionality._ import io.viash.runners._ @@ -60,7 +62,7 @@ final case class CollectedSchemas ( object CollectedSchemas { implicit class RichSymbol(s: Symbol) { - def shortName = s.fullName.split('.').last + def shortName = ""//s.fullName.split('.').last } case class MemberInfo ( @@ -69,7 +71,7 @@ object CollectedSchemas { className: String, inheritanceIndex: Int ) { - def fullName = symbol.fullName + def fullName = symbol.name//fullName def shortName = symbol.shortName } @@ -83,41 +85,42 @@ object CollectedSchemas { private implicit val encodeDeprecatedOrRemoved: Encoder.AsObject[DeprecatedOrRemovedSchema] = deriveConfiguredEncoder private implicit val encodeExample: Encoder.AsObject[ExampleSchema] = deriveConfiguredEncoder - private def getMembers[T: TypeTag](): (Map[String,List[MemberInfo]], List[Symbol]) = { - - val name = typeOf[T].typeSymbol.shortName - - // Get all members and filter for constructors, first one should be the best (most complete) one - // Traits don't have constructors - // Get all parameters and store their short name - val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String]) - - val baseClasses = typeOf[T].baseClasses - .filter(_.fullName.startsWith("io.viash")) - - // If we're only getting a abstract class/trait, not a final implementation, use these definitions (otherwise we're left with nothing). - val documentFully = - baseClasses.length == 1 && - baseClasses.head.isAbstract && - baseClasses.head.annotations.exists(a => a.tree.tpe =:= typeOf[documentFully]) - - val memberNames = typeOf[T].members - .filter(!_.isMethod || documentFully) - .map(_.shortName) - .toSeq - - val allMembers = baseClasses - .zipWithIndex - .flatMap{ case (baseClass, index) => - baseClass.info.members - .filter(_.fullName.startsWith("io.viash")) - .filter(m => memberNames.contains(m.shortName)) - .filter(m => !m.info.getClass.toString.endsWith("NullaryMethodType") || index != 0 || documentFully) // Only regular members if base class, otherwise all members - .map(y => MemberInfo(y, (constructorMembers.contains(y.shortName)), baseClass.fullName, index)) - } - .groupBy(k => k.shortName) + private def getMembers[T/*: TypeTag*/](): (Map[String,List[MemberInfo]], List[Symbol]) = { + (Map.empty, Nil) + + // val name = typeOf[T].typeSymbol.shortName + + // // Get all members and filter for constructors, first one should be the best (most complete) one + // // Traits don't have constructors + // // Get all parameters and store their short name + // val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String]) + + // val baseClasses = typeOf[T].baseClasses + // .filter(_.fullName.startsWith("io.viash")) + + // // If we're only getting a abstract class/trait, not a final implementation, use these definitions (otherwise we're left with nothing). + // val documentFully = + // baseClasses.length == 1 && + // baseClasses.head.isAbstract && + // baseClasses.head.annotations.exists(a => a.tree.tpe =:= typeOf[documentFully]) + + // val memberNames = typeOf[T].members + // .filter(!_.isMethod || documentFully) + // .map(_.shortName) + // .toSeq + + // val allMembers = baseClasses + // .zipWithIndex + // .flatMap{ case (baseClass, index) => + // baseClass.info.members + // .filter(_.fullName.startsWith("io.viash")) + // .filter(m => memberNames.contains(m.shortName)) + // .filter(m => !m.info.getClass.toString.endsWith("NullaryMethodType") || index != 0 || documentFully) // Only regular members if base class, otherwise all members + // .map(y => MemberInfo(y, (constructorMembers.contains(y.shortName)), baseClass.fullName, index)) + // } + // .groupBy(k => k.shortName) - (allMembers, baseClasses) + // (allMembers, baseClasses) } lazy val schemaClasses = List( @@ -204,60 +207,62 @@ object CollectedSchemas { .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") } - private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]) = { - val annMembers = members - .map{ case (memberName, memberInfo) => { - val h = memberInfo.head - val annotations = memberInfo.flatMap(_.symbol.annotations) - (h.fullName, h.symbol.info.toString, annotations, h.className, h.inheritanceIndex, Nil) - } } - .filter(_._3.length > 0) - val annThis = ("__this__", classes.head.name.toString(), classes.head.annotations, "", 0, classes.map(_.fullName)) - val allAnnotations = annThis :: annMembers.toList - allAnnotations - .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class - } - - private val getSchema = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { - case (members, classes) => { - annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } - } - } + // private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]) = { + // val annMembers = members + // .map{ case (memberName, memberInfo) => { + // val h = memberInfo.head + // val annotations = memberInfo.flatMap(_.symbol.annotations) + // (h.fullName, h.symbol.info.toString, annotations, h.className, h.inheritanceIndex, Nil) + // } } + // .filter(_._3.length > 0) + // val annThis = ("__this__", classes.head.name.toString(), classes.head.annotations, "", 0, classes.map(_.fullName)) + // val allAnnotations = annThis :: annMembers.toList + // allAnnotations + // .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class + // } + + // private val getSchema = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { + // case (members, classes) => { + // annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } + // } + // } // get all parameters for a given type, including parent class annotations - def getParameters[T: TypeTag]() = getSchema(getMembers[T]()) + // def getParameters[T: TypeTag]() = getSchema(getMembers[T]()) // Main call for documentation output - lazy val fullData: List[List[ParameterSchema]] = schemaClasses.map{ v => getSchema(v)} + lazy val fullData: List[List[ParameterSchema]] = Nil//schemaClasses.map{ v => getSchema(v)} lazy val data: List[List[ParameterSchema]] = fullData.map(_.filter(p => !p.hasUndocumented && !p.hasInternalFunctionality)) def getKeyFromParamList(data: List[ParameterSchema]): String = data.find(p => p.name == "__this__").get.`type` def getJson: Json = data.asJson - private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = { - val issueMembers = members - .toList - .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement. - .map{ case (k, v) => (k, v.map(_.symbol.annotations.length).sum) } // (name, # annotations) - .filter(_._2 == 0) - .map(_._1) + // private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = { + // val issueMembers = members + // .toList + // .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement. + // .map{ case (k, v) => (k, v.map(_.symbol.annotations.length).sum) } // (name, # annotations) + // .filter(_._2 == 0) + // .map(_._1) - val ownClassArr = if (classes.head.annotations.length == 0) Seq("__this__") else Nil - issueMembers ++ ownClassArr - } + // val ownClassArr = if (classes.head.annotations.length == 0) Seq("__this__") else Nil + // issueMembers ++ ownClassArr + // } def getMemberName(members: Map[String,List[MemberInfo]], classes: List[Symbol]): String = classes.head.shortName // Main call for checking whether all arguments are annotated // Add extra non-annotated value so we can always somewhat check the code is functional - def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap { - v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) - }.toMap + // def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap { + // v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) + // }.toMap + def getAllNonAnnotated: Map[String, String] = Map.empty def getAllDeprecations: Map[String, DeprecatedOrRemovedSchema] = { - val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap - arr.filter(t => t._2.isDefined).map(t => (t._1, t._2.get)) + // val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap + // arr.filter(t => t._2.isDefined).map(t => (t._1, t._2.get)) + Map.empty } } diff --git a/src/main/scala/io/viash/schemas/JsonSchema.scala b/src/main/scala/io/viash/schemas/JsonSchema.scala index 2f74a5570..a36185e91 100644 --- a/src/main/scala/io/viash/schemas/JsonSchema.scala +++ b/src/main/scala/io/viash/schemas/JsonSchema.scala @@ -119,128 +119,129 @@ object JsonSchema { data.find(_.name == "__this__").get def createSchema(info: List[ParameterSchema])(implicit config: SchemaConfig): (String, Json) = { - - def removeMarkup(text: String): String = { - val markupRegex = raw"@\[(.*?)\]\(.*?\)".r - val backtickRegex = "`(\"[^`\"]*?\")`".r - val textWithoutMarkup = markupRegex.replaceAllIn(text, "$1") - backtickRegex.replaceAllIn(textWithoutMarkup, "$1") - } - - val thisParameter = getThisParameter(info) - val description = removeMarkup(thisParameter.description.get) - val subclass = thisParameter.subclass.map(l => l.head) - val properties = - info - .filter(p => !p.name.startsWith("__")) // remove __this__ - .filter(p => !p.removed.isDefined && (!config.strict || !p.deprecated.isDefined)) // always remove 'removed' arguments and if need be create a strict schema by removing deprecated - .filter(p => !config.strict || !(p.name == "arguments" && (thisParameter.`type` == "Functionality" || thisParameter.`type` == "Config"))) // exception: remove 'arguments' in 'Functionality' for strict schema - val propertiesJson = properties.map(p => { - val pDescription = p.description.map(s => removeMarkup(s)) - val trimmedType = p.`type` match { - case s if s.startsWith("Option[") => s.stripPrefix("Option[").stripSuffix("]") - case s => s - } - - val mapRegex = "(List)?Map\\[String,(\\w*)\\]".r - - implicit val useAllInEither = thisParameter.`type` == "NextflowDirectives" || thisParameter.`type` == "NextflowAuto" - - trimmedType match { - case s"List[$s]" => - (p.name, arrayType(s, pDescription)) - - case "Either[String,List[String]]" => - (p.name, eitherJson( - valueType("String", pDescription), - arrayType("String", pDescription) - )) - - case "Either[Map[String,String],String]" => - (p.name, eitherJson( - mapType("String", pDescription), - valueType("String", pDescription) - )) - - case s"Either[$s,$t]" => - (p.name, eitherJson( - valueType(s, pDescription), - valueType(t, pDescription) - )) - - case "OneOrMore[Map[String,String]]" => - (p.name, oneOrMoreJson( - mapType("String", pDescription) - )) - - case "OneOrMore[Either[String,Map[String,String]]]" => - (p.name, oneOrMoreJson( - eitherJson( - valueType("String", pDescription), - mapType("String", pDescription) - ) - )) - - case s"OneOrMore[$s]" => - if (s == "String" && p.name == "port" && subclass == Some("executable")) { - // Custom exception - // This is the port field for a excutable runner. - // We want to allow a Strings or Ints. - (p.name, eitherJson( - valueType("Int", pDescription), - valueType("String", pDescription), - arrayType("Int", pDescription), - arrayType("String", pDescription) - )) - } else { - (p.name, oneOrMoreType(s, pDescription)) - } - - case mapRegex(_, s) => - (p.name, mapType(s, pDescription)) - - case s if p.name == "type" && subclass.isDefined => - if (config.minimal) { - ("type", Json.obj( - "const" -> Json.fromString(subclass.get) - )) - } else { - ("type", Json.obj( - "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class - "const" -> Json.fromString(subclass.get) - )) - } + ("", Json.Null) + + // def removeMarkup(text: String): String = { + // val markupRegex = raw"@\[(.*?)\]\(.*?\)".r + // val backtickRegex = "`(\"[^`\"]*?\")`".r + // val textWithoutMarkup = markupRegex.replaceAllIn(text, "$1") + // backtickRegex.replaceAllIn(textWithoutMarkup, "$1") + // } + + // val thisParameter = getThisParameter(info) + // val description = removeMarkup(thisParameter.description.get) + // val subclass = thisParameter.subclass.map(l => l.head) + // val properties = + // info + // .filter(p => !p.name.startsWith("__")) // remove __this__ + // .filter(p => !p.removed.isDefined && (!config.strict || !p.deprecated.isDefined)) // always remove 'removed' arguments and if need be create a strict schema by removing deprecated + // .filter(p => !config.strict || !(p.name == "arguments" && (thisParameter.`type` == "Functionality" || thisParameter.`type` == "Config"))) // exception: remove 'arguments' in 'Functionality' for strict schema + // val propertiesJson = properties.map(p => { + // val pDescription = p.description.map(s => removeMarkup(s)) + // val trimmedType = p.`type` match { + // case s if s.startsWith("Option[") => s.stripPrefix("Option[").stripSuffix("]") + // case s => s + // } + + // val mapRegex = "(List)?Map\\[String,(\\w*)\\]".r + + // implicit val useAllInEither = thisParameter.`type` == "NextflowDirectives" || thisParameter.`type` == "NextflowAuto" + + // trimmedType match { + // case s"List[$s]" => + // (p.name, arrayType(s, pDescription)) + + // case "Either[String,List[String]]" => + // (p.name, eitherJson( + // valueType("String", pDescription), + // arrayType("String", pDescription) + // )) + + // case "Either[Map[String,String],String]" => + // (p.name, eitherJson( + // mapType("String", pDescription), + // valueType("String", pDescription) + // )) + + // case s"Either[$s,$t]" => + // (p.name, eitherJson( + // valueType(s, pDescription), + // valueType(t, pDescription) + // )) + + // case "OneOrMore[Map[String,String]]" => + // (p.name, oneOrMoreJson( + // mapType("String", pDescription) + // )) + + // case "OneOrMore[Either[String,Map[String,String]]]" => + // (p.name, oneOrMoreJson( + // eitherJson( + // valueType("String", pDescription), + // mapType("String", pDescription) + // ) + // )) + + // case s"OneOrMore[$s]" => + // if (s == "String" && p.name == "port" && subclass == Some("executable")) { + // // Custom exception + // // This is the port field for a excutable runner. + // // We want to allow a Strings or Ints. + // (p.name, eitherJson( + // valueType("Int", pDescription), + // valueType("String", pDescription), + // arrayType("Int", pDescription), + // arrayType("String", pDescription) + // )) + // } else { + // (p.name, oneOrMoreType(s, pDescription)) + // } + + // case mapRegex(_, s) => + // (p.name, mapType(s, pDescription)) + + // case s if p.name == "type" && subclass.isDefined => + // if (config.minimal) { + // ("type", Json.obj( + // "const" -> Json.fromString(subclass.get) + // )) + // } else { + // ("type", Json.obj( + // "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class + // "const" -> Json.fromString(subclass.get) + // )) + // } - case s => - (p.name, valueType(s, pDescription)) - } - - }) - - val required = properties.filter(p => - !( - p.`type`.startsWith("Option[") || - p.default.isDefined || - p.hasUndocumented || - (p.name == "type" && thisParameter.`type` == "PlainFile" && !config.strict) // Custom exception, file resources are "kind of" default - ) || - // Strict schema is what will be outputted by Viash, so fields with a default value other than 'None' will always have a value -> add it in the strict schema as required. - (!p.`type`.startsWith("Option[") && p.default != Some("Empty") && config.strict && !p.hasUndocumented)) - val requiredJson = required.map(p => Json.fromString(p.name)) - - val k = thisParameter.`type` - val descr = config.minimal match { - case true => None - case false => Some(description) - } - val v = Json.obj( - descr.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ - Seq("type" -> Json.fromString("object"), - "properties" -> Json.obj(propertiesJson: _*), - "required" -> Json.arr(requiredJson: _*), - "additionalProperties" -> Json.False): _* - ) - k -> v + // case s => + // (p.name, valueType(s, pDescription)) + // } + + // }) + + // val required = properties.filter(p => + // !( + // p.`type`.startsWith("Option[") || + // p.default.isDefined || + // p.hasUndocumented || + // (p.name == "type" && thisParameter.`type` == "PlainFile" && !config.strict) // Custom exception, file resources are "kind of" default + // ) || + // // Strict schema is what will be outputted by Viash, so fields with a default value other than 'None' will always have a value -> add it in the strict schema as required. + // (!p.`type`.startsWith("Option[") && p.default != Some("Empty") && config.strict && !p.hasUndocumented)) + // val requiredJson = required.map(p => Json.fromString(p.name)) + + // val k = thisParameter.`type` + // val descr = config.minimal match { + // case true => None + // case false => Some(description) + // } + // val v = Json.obj( + // descr.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ + // Seq("type" -> Json.fromString("object"), + // "properties" -> Json.obj(propertiesJson: _*), + // "required" -> Json.arr(requiredJson: _*), + // "additionalProperties" -> Json.False): _* + // ) + // k -> v } def createSuperClassSchema(info: List[ParameterSchema])(implicit config: SchemaConfig): (String, Json) = { diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index ee4d847ca..f0585d43b 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -17,7 +17,7 @@ package io.viash.schemas -import scala.reflect.runtime.universe._ +// import scala.reflect.runtime.universe._ import io.viash.schemas.internalFunctionality final case class ParameterSchema( @@ -44,125 +44,125 @@ object ParameterSchema { s.replaceAll("\\\\n", "\n").stripMargin(marginChar) } - private def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { - l.map(i => i match { - case Literal(Constant(value: String)) => - unfinishedStringStripMargin(value, marginChar) - case _ => - "unmatched in mapTreeList: " + i.toString() - }).mkString - } + // private def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { + // l.map(i => i match { + // case Literal(Constant(value: String)) => + // unfinishedStringStripMargin(value, marginChar) + // case _ => + // "unmatched in mapTreeList: " + i.toString() + // }).mkString + // } // Traverse tree information and extract values or lists of values - private def annotationToStrings(ann: Annotation):(String, List[String]) = { - val name = ann.tree.tpe.toString() - val values = ann.tree match { - case Apply(c, args: List[Tree]) => - args.collect({ - case i: Tree => - i match { - // Here 'Apply' contains lists - // While 'Select' has a single element - case Literal(Constant(value: String)) => - value - // case Select(Select(a, b), stripMargin) => - // unfinishedStringStripMargin(b) - case Select(Apply(a, a2), b) if b.toString == "stripMargin" => - mapTreeList(a2) - case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => - val stripper = stripMargin.head.toString.charAt(1) - mapTreeList(a2, stripper) - case _ => - "unmatched in annotationToStrings: " + i.toString() - } - }) - } - (name, values) - } - - def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): ParameterSchema = { - - def beautifyTypeName(s: String): String = { - - // "tpe[a]" -> "(\w*)\[(\w*)\]" - def regexify(s: String) = s.replace("[", "\\[").replace("]", "\\]").replaceAll("\\w+", "(\\\\w+)").r - - val regex0 = regexify("tpe") - val regex1 = regexify("tpe[a]") - val regex2 = regexify("tpe[a,b]") - val regexNested1 = regexify("tpe[a[b,c]]") - val regexNested2 = regexify("tpe[a[b,c[d,e]]]") - val regexNested3 = regexify("tpe[a,b[c,d]]") - val regexNested4 = regexify("tpe[a[b[c,d],e]]") - val regexNested5 = regexify("tpe[a[b,c[d]]]") - val regexNested6 = regexify("tpe[a[b,c],d]") - val regexNested7 = regexify("tpe[a,b[c]]") - - def map(a: String, b: String): String = s"Map of $a to $b" - def either(a: String, b: String): String = s"""Either $a or $b""" + // private def annotationToStrings(ann: Annotation):(String, List[String]) = { + // val name = ann.tree.tpe.toString() + // val values = ann.tree match { + // case Apply(c, args: List[Tree]) => + // args.collect({ + // case i: Tree => + // i match { + // // Here 'Apply' contains lists + // // While 'Select' has a single element + // case Literal(Constant(value: String)) => + // value + // // case Select(Select(a, b), stripMargin) => + // // unfinishedStringStripMargin(b) + // case Select(Apply(a, a2), b) if b.toString == "stripMargin" => + // mapTreeList(a2) + // case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => + // val stripper = stripMargin.head.toString.charAt(1) + // mapTreeList(a2, stripper) + // case _ => + // "unmatched in annotationToStrings: " + i.toString() + // } + // }) + // } + // (name, values) + // } + + // def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): ParameterSchema = { + + // def beautifyTypeName(s: String): String = { + + // // "tpe[a]" -> "(\w*)\[(\w*)\]" + // def regexify(s: String) = s.replace("[", "\\[").replace("]", "\\]").replaceAll("\\w+", "(\\\\w+)").r + + // val regex0 = regexify("tpe") + // val regex1 = regexify("tpe[a]") + // val regex2 = regexify("tpe[a,b]") + // val regexNested1 = regexify("tpe[a[b,c]]") + // val regexNested2 = regexify("tpe[a[b,c[d,e]]]") + // val regexNested3 = regexify("tpe[a,b[c,d]]") + // val regexNested4 = regexify("tpe[a[b[c,d],e]]") + // val regexNested5 = regexify("tpe[a[b,c[d]]]") + // val regexNested6 = regexify("tpe[a[b,c],d]") + // val regexNested7 = regexify("tpe[a,b[c]]") + + // def map(a: String, b: String): String = s"Map of $a to $b" + // def either(a: String, b: String): String = s"""Either $a or $b""" - s match { - case regex0(tpe) => s"$tpe" - case regex1(tpe, subtpe) => s"$tpe of $subtpe" - case regex2("Map", subtpe1, subtpe2) => map(subtpe1,subtpe2) - case regex2("ListMap", subtpe1, subtpe2) => map(subtpe1, subtpe2) - case regex2("Either", subtpe1, subtpe2) => either(subtpe1, subtpe2) - case regexNested1(tpe, a, b, c) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c]")}" - case regexNested2(tpe, a, b ,c ,d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d,$e]]")}" - case regexNested3("Either", a, b, c, d) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c,$d]")) - case regexNested4(tpe, a, b, c, d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b[$c,$d],$e]")}" - case regexNested5(tpe, a, b, c, d) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d]]")}" - case regexNested6("Either", a, b, c, d) => either(beautifyTypeName(s"$a[$b,$c]"), beautifyTypeName(d)) - case regexNested7("Either", a, b, c) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c]")) - case _ => s - } - } - - val annStrings = annotations.map(annotationToStrings(_)) - val hierarchyOption = hierarchy match { - case l if l.length > 0 => Some(l) - case _ => None - } - - // name is e.g. "io.viash.config.Config.name", only keep "name" - // name can also be "__this__" - // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. - val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) - val nameFromClass = name.split('.').last - val name_ = (nameOverride, nameFromClass) match { - case (Some(_), "__this__") => "__this__" - case (Some(ann), _) => ann - case (None, name) => name - } - - val typeName = (`type`, nameOverride, nameFromClass) match { - case (_, Some(newTypeName), "__this__") => newTypeName - case (typeName, _, _) => typeName - } + // s match { + // case regex0(tpe) => s"$tpe" + // case regex1(tpe, subtpe) => s"$tpe of $subtpe" + // case regex2("Map", subtpe1, subtpe2) => map(subtpe1,subtpe2) + // case regex2("ListMap", subtpe1, subtpe2) => map(subtpe1, subtpe2) + // case regex2("Either", subtpe1, subtpe2) => either(subtpe1, subtpe2) + // case regexNested1(tpe, a, b, c) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c]")}" + // case regexNested2(tpe, a, b ,c ,d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d,$e]]")}" + // case regexNested3("Either", a, b, c, d) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c,$d]")) + // case regexNested4(tpe, a, b, c, d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b[$c,$d],$e]")}" + // case regexNested5(tpe, a, b, c, d) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d]]")}" + // case regexNested6("Either", a, b, c, d) => either(beautifyTypeName(s"$a[$b,$c]"), beautifyTypeName(d)) + // case regexNested7("Either", a, b, c) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c]")) + // case _ => s + // } + // } + + // val annStrings = annotations.map(annotationToStrings(_)) + // val hierarchyOption = hierarchy match { + // case l if l.length > 0 => Some(l) + // case _ => None + // } + + // // name is e.g. "io.viash.config.Config.name", only keep "name" + // // name can also be "__this__" + // // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. + // val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) + // val nameFromClass = name.split('.').last + // val name_ = (nameOverride, nameFromClass) match { + // case (Some(_), "__this__") => "__this__" + // case (Some(ann), _) => ann + // case (None, name) => name + // } + + // val typeName = (`type`, nameOverride, nameFromClass) match { + // case (_, Some(newTypeName), "__this__") => newTypeName + // case (typeName, _, _) => typeName + // } - val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head}) - val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) - val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) - val examples = example ::: exampleWithDescription match { - case l if l.length > 0 => Some(l) - case _ => None - } - val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head}) - val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) - val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) - val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head}) - val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") - val default = defaultFromAnnotation orElse defaultFromType - val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { - case l if l.nonEmpty => Some(l) - case _ => None - } + // val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head}) + // val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) + // val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) + // val examples = example ::: exampleWithDescription match { + // case l if l.length > 0 => Some(l) + // case _ => None + // } + // val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head}) + // val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) + // val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) + // val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head}) + // val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") + // val default = defaultFromAnnotation orElse defaultFromType + // val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { + // case l if l.nonEmpty => Some(l) + // case _ => None + // } - val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")} - val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")} + // val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")} + // val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")} - ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass, undocumented, internalFunctionality) - } + // ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass, undocumented, internalFunctionality) + // } } final case class DeprecatedOrRemovedSchema( diff --git a/src/main/scala/io/viash/wrapper/BashWrapper.scala b/src/main/scala/io/viash/wrapper/BashWrapper.scala index 0faefc48f..60554a087 100644 --- a/src/main/scala/io/viash/wrapper/BashWrapper.scala +++ b/src/main/scala/io/viash/wrapper/BashWrapper.scala @@ -25,6 +25,7 @@ import java.nio.file.Paths import io.viash.ViashNamespace import io.viash.config.arguments._ import io.viash.config.resources.Executable +import io.viash.helpers.data_structures.oneOrMoreToList object BashWrapper { val metaArgs: List[Argument[_]] = { @@ -571,13 +572,25 @@ object BashWrapper { | ViashWarning '${param.name}' specifies a maximum value but the value was not verified as neither \\'bc\\' or \\'awk\\' are present on the system. | fi |""".stripMargin - def minCheckInt(min: Long) = + def minCheckInt(min: Int) = s""" if [[ $$${param.VIASH_PAR} -lt $min ]]; then | ViashError '${param.name}' has be more than or equal to $min. Use "--help" to get more information on the parameters. | exit 1 | fi |""".stripMargin - def maxCheckInt(max: Long) = + def maxCheckInt(max: Int) = + s""" if [[ $$${param.VIASH_PAR} -gt $max ]]; then + | ViashError '${param.name}' has be less than or equal to $max. Use "--help" to get more information on the parameters. + | exit 1 + | fi + |""".stripMargin + def minCheckLong(min: Long) = + s""" if [[ $$${param.VIASH_PAR} -lt $min ]]; then + | ViashError '${param.name}' has be more than or equal to $min. Use "--help" to get more information on the parameters. + | exit 1 + | fi + |""".stripMargin + def maxCheckLong(max: Long) = s""" if [[ $$${param.VIASH_PAR} -gt $max ]]; then | ViashError '${param.name}' has be less than or equal to $max. Use "--help" to get more information on the parameters. | exit 1 @@ -586,13 +599,13 @@ object BashWrapper { val minCheck = param match { case p: IntegerArgument if min.isDefined => minCheckInt(min.get) - case p: LongArgument if min.isDefined => minCheckInt(min.get) + case p: LongArgument if min.isDefined => minCheckLong(min.get) case p: DoubleArgument if min.isDefined => minCheckDouble(min.get) case _ => "" } val maxCheck = param match { case p: IntegerArgument if max.isDefined => maxCheckInt(max.get) - case p: LongArgument if max.isDefined => maxCheckInt(max.get) + case p: LongArgument if max.isDefined => maxCheckLong(max.get) case p: DoubleArgument if max.isDefined => maxCheckDouble(max.get) case _ => "" } From f9226ab812919292ab1ba4f4aeb664dbcefb19f3 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 29 Aug 2024 14:36:24 +0200 Subject: [PATCH 02/46] Start re-enabling circe calls with replaced encoder and decoders still need to tweak methods a bit but it's progress --- .../viash/config/dependencies/package.scala | 68 +++++++++---------- src/main/scala/io/viash/config/package.scala | 19 +++--- .../DeriveConfiguredDecoderFullChecks.scala | 8 ++- ...ConfiguredDecoderWithValidationCheck.scala | 7 +- .../circe/DeriveConfiguredEncoder.scala | 26 +++++++ .../circe/DeriveConfiguredEncoderStrict.scala | 11 +-- .../io/viash/helpers/circe/package.scala | 21 +++--- .../viash/packageConfig/PackageConfig.scala | 2 +- .../io/viash/packageConfig/package.scala | 12 ++-- .../io/viash/schemas/CollectedSchemas.scala | 1 - 10 files changed, 107 insertions(+), 68 deletions(-) create mode 100644 src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index e07755845..d1a8cbef2 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -19,7 +19,7 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} // import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -// import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ +import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import cats.syntax.functor._ import dependencies.GithubRepository @@ -51,21 +51,21 @@ package object dependencies { // objJson deepMerge typeJson // } - // implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoderStrict - // implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoderStrict - // implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoderStrict - // implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoderStrict - // implicit def encodeRepositoryWithName[A <: RepositoryWithName]: Encoder[A] = Encoder.instance { - // par => - // val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - // val objJson = par match { - // case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) - // case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) - // case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) - // case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) - // } - // objJson deepMerge typeJson - // } + implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoderStrict + implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoderStrict + implicit val encodeViashhubRepositoryWithName: Encoder.AsObject[ViashhubRepositoryWithName] = deriveConfiguredEncoderStrict + implicit val encodeLocalRepositoryWithName: Encoder.AsObject[LocalRepositoryWithName] = deriveConfiguredEncoderStrict + implicit def encodeRepositoryWithName[A <: RepositoryWithName]: Encoder[A] = Encoder.instance { + par => + val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + val objJson = par match { + case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) + case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) + case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) + case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) + } + objJson deepMerge typeJson + } // implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks @@ -89,24 +89,24 @@ package object dependencies { // decoder(cursor) // } - // implicit val decodeGitRepositoryWithName: Decoder[GitRepositoryWithName] = deriveConfiguredDecoderFullChecks - // implicit val decodeGithubRepositoryWithName: Decoder[GithubRepositoryWithName] = deriveConfiguredDecoderFullChecks - // implicit val decodeViashhubRepositoryWithName: Decoder[ViashhubRepositoryWithName] = deriveConfiguredDecoderFullChecks - // implicit val decodeLocalRepositoryWithName: Decoder[LocalRepositoryWithName] = deriveConfiguredDecoderFullChecks - // implicit def decodeRepositoryWithName: Decoder[RepositoryWithName] = Decoder.instance { - // cursor => - // val decoder: Decoder[RepositoryWithName] = - // cursor.downField("type").as[String] match { - // case Right("git") => decodeGitRepositoryWithName.widen - // case Right("github") => decodeGithubRepositoryWithName.widen - // case Right("vsh") => decodeViashhubRepositoryWithName.widen - // case Right("local") => decodeLocalRepositoryWithName.widen - // case Right(typ) => - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepositoryWithName](typ, List("git", "github", "vsh", "local")).widen - // case Left(exception) => throw exception - // } + implicit val decodeGitRepositoryWithName: Decoder[GitRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit val decodeGithubRepositoryWithName: Decoder[GithubRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit val decodeViashhubRepositoryWithName: Decoder[ViashhubRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit val decodeLocalRepositoryWithName: Decoder[LocalRepositoryWithName] = deriveConfiguredDecoderFullChecks + implicit def decodeRepositoryWithName: Decoder[RepositoryWithName] = Decoder.instance { + cursor => + val decoder: Decoder[RepositoryWithName] = + cursor.downField("type").as[String] match { + case Right("git") => decodeGitRepositoryWithName.widen + case Right("github") => decodeGithubRepositoryWithName.widen + case Right("vsh") => decodeViashhubRepositoryWithName.widen + case Right("local") => decodeLocalRepositoryWithName.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepositoryWithName](typ, List("git", "github", "vsh", "local")).widen + case Left(exception) => throw exception + } - // decoder(cursor) - // } + decoder(cursor) + } } diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index bf119c06d..af5dfa11b 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -26,6 +26,7 @@ import io.circe.{Decoder, Encoder, Json, HCursor, JsonObject} // import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoderWithValidationCheck // import io.viash.platforms.decodePlatform // import io.viash.functionality.decodeFunctionality +import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.exceptions.ConfigParserValidationException import config.ArgumentGroup @@ -38,10 +39,10 @@ import config.arguments._ package object config { import io.viash.helpers.circe._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // encoders and decoders for Config // implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] @@ -276,9 +277,9 @@ package object config { // implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder // implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks - // encoder and decoder for Author - // implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder - // implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks + // encoder and decoder for Author + implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder + implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks // encoder and decoder for Requirements // implicit val encodeComputationalRequirements: Encoder.AsObject[ComputationalRequirements] = deriveConfiguredEncoder @@ -294,9 +295,9 @@ package object config { // _.withFocus(_.mapString(_.toLowerCase())) // } - // implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict - // implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks + implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict + implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks - // implicit val encodeReferences: Encoder.AsObject[References] = deriveConfiguredEncoderStrict - // implicit val decodeReferences: Decoder[References] = deriveConfiguredDecoderFullChecks + implicit val encodeReferences: Encoder.AsObject[References] = deriveConfiguredEncoderStrict + implicit val decodeReferences: Decoder[References] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala index d4cfc7cb0..6dee1b41c 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -20,9 +20,9 @@ package io.viash.helpers.circe // import shapeless.Lazy // import scala.reflect.runtime.universe._ -// import io.circe.Decoder -// import io.circe.generic.extras.decoding.ConfiguredDecoder -// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +import io.circe.Decoder +import io.circe.derivation.Configuration +import scala.deriving.Mirror object DeriveConfiguredDecoderFullChecks { // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ @@ -34,4 +34,6 @@ object DeriveConfiguredDecoderFullChecks { // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." // ) // .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] ) + + inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = Decoder.derivedConfigured[A] } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 5e62c2d0f..7bb3ee61b 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -70,8 +70,8 @@ object DeriveConfiguredDecoderWithValidationCheck { // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." // ) - // // Dummy decoder to generate exceptions when an invalid type is specified - // // We need a valid class type to be specified + // Dummy decoder to generate exceptions when an invalid type is specified + // We need a valid class type to be specified // def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] // .validate( // pred => { @@ -80,5 +80,8 @@ object DeriveConfiguredDecoderWithValidationCheck { // }, // s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." // ) + def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String]): Decoder[A] = Decoder.instance { cursor => + throw new ConfigParserSubTypeException(tpe, validTypes, cursor.value.toString()) + } } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala new file mode 100644 index 000000000..34d2ab77a --- /dev/null +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers.circe + +import io.circe.Encoder +import io.circe.derivation.{Configuration, ConfiguredEncoder} +import scala.deriving.Mirror + +object DeriveConfiguredEncoder { + inline def deriveConfiguredEncoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A] +} \ No newline at end of file diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala index 47fd85888..b299aae75 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala @@ -17,10 +17,11 @@ package io.viash.helpers.circe -import io.circe.{Encoder, Json, HCursor} -// import io.circe.generic.extras.Configuration -// import io.circe.generic.extras.semiauto.deriveConfiguredEncoder -// import io.circe.generic.extras.encoding.ConfiguredAsObjectEncoder +import io.circe.Encoder +import io.circe.derivation.{Configuration, ConfiguredEncoder} +import scala.deriving.Mirror + +// import io.circe.{Encoder, Json, HCursor} // import scala.reflect.runtime.universe._ // import shapeless.Lazy @@ -39,4 +40,6 @@ object DeriveConfiguredEncoderStrict { // .getOrElse(true) // fallback, shouldn't really happen // ) // } + + inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A] } diff --git a/src/main/scala/io/viash/helpers/circe/package.scala b/src/main/scala/io/viash/helpers/circe/package.scala index 19288a91b..46fad0733 100644 --- a/src/main/scala/io/viash/helpers/circe/package.scala +++ b/src/main/scala/io/viash/helpers/circe/package.scala @@ -18,23 +18,24 @@ package io.viash.helpers import io.circe._ -// import io.circe.generic.extras.Configuration +import io.circe.derivation.Configuration import java.net.URI import data_structures.OneOrMore import java.nio.file.Paths package object circe { - // implicit val customConfig: Configuration = - // Configuration.default.withDefaults.withStrictDecoding + implicit val customConfig: Configuration = + Configuration.default.withDefaults.withStrictDecoding - // encoder and decoder for Either - implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { - _.fold(ea(_), eb(_)) - } - implicit def decodeEither[A,B](implicit a: Decoder[A], b: Decoder[B]): Decoder[Either[A,B]] = { - a either b - } + // // encoder and decoder for Either + // implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { + // _.fold(ea(_), eb(_)) + // } + + // implicit def decodeEither[A,B](implicit a: Decoder[A], b: Decoder[B]): Decoder[Either[A,B]] = { + // a either b + // } // encoder and decoder for OneOrMore implicit def encodeOneOrMore[A](implicit enc: Encoder[List[A]]): Encoder[OneOrMore[A]] = { diff --git a/src/main/scala/io/viash/packageConfig/PackageConfig.scala b/src/main/scala/io/viash/packageConfig/PackageConfig.scala index fa216a7f0..0178eee98 100644 --- a/src/main/scala/io/viash/packageConfig/PackageConfig.scala +++ b/src/main/scala/io/viash/packageConfig/PackageConfig.scala @@ -246,7 +246,7 @@ object PackageConfig { /* PACKAGE 0: converted from json */ // convert Json into ViashPackage - val pack0 = PackageConfig()//Convert.jsonToClass[PackageConfig](json, path.toString()) + val pack0 = Convert.jsonToClass[PackageConfig](json, path.toString()) /* PACKAGE 1: make resources absolute */ // make paths absolute diff --git a/src/main/scala/io/viash/packageConfig/package.scala b/src/main/scala/io/viash/packageConfig/package.scala index 6654d2fe4..d33b3e5b6 100644 --- a/src/main/scala/io/viash/packageConfig/package.scala +++ b/src/main/scala/io/viash/packageConfig/package.scala @@ -21,9 +21,13 @@ import io.circe.{Decoder, Encoder} package object packageConfig { import io.viash.helpers.circe._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.config.{decodeAuthor, encodeAuthor} + import io.viash.config.{decodeLinks, encodeLinks} + import io.viash.config.{decodeReferences, encodeReferences} + import io.viash.config.dependencies.{decodeRepositoryWithName, encodeRepositoryWithName} - // implicit val encodePackageConfig: Encoder.AsObject[PackageConfig] = deriveConfiguredEncoderStrict - // implicit val decodePackageConfig: Decoder[PackageConfig] = deriveConfiguredDecoderFullChecks + implicit val encodePackageConfig: Encoder.AsObject[PackageConfig] = deriveConfiguredEncoderStrict + implicit val decodePackageConfig: Decoder[PackageConfig] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index d5b49286b..8b6fa89fd 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -22,7 +22,6 @@ import io.circe.{Encoder, Printer => JsonPrinter} import io.circe.syntax.EncoderOps // import io.circe.generic.extras.semiauto.deriveConfiguredEncoder import io.circe.generic.semiauto.{deriveEncoder as deriveConfiguredEncoder} -import io.circe.generic.semiauto.{deriveEncoder as deriveConfiguredEncoderStrict} import io.viash.functionality._ import io.viash.runners._ From 673570b64d27800014f52a15dfedb33c39dadf24 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 29 Aug 2024 16:33:26 +0200 Subject: [PATCH 03/46] Start to re-enable decoders and encoders --- src/main/scala/io/viash/config/Config.scala | 2 +- src/main/scala/io/viash/config/Status.scala | 13 +- .../viash/config/dependencies/package.scala | 81 +++++----- src/main/scala/io/viash/config/package.scala | 24 ++- .../io/viash/config/resources/package.scala | 144 +++++++++--------- .../io/viash/engines/docker/package.scala | 24 +-- src/main/scala/io/viash/engines/package.scala | 63 ++++---- .../viash/engines/requirements/package.scala | 78 +++++----- .../io/viash/functionality/package.scala | 10 +- .../scala/io/viash/platforms/package.scala | 68 +++++---- .../io/viash/runners/executable/package.scala | 24 +-- .../io/viash/runners/nextflow/package.scala | 28 +++- src/main/scala/io/viash/runners/package.scala | 64 ++++---- 13 files changed, 335 insertions(+), 288 deletions(-) diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index 1005cf2f2..e2273b030 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -657,7 +657,7 @@ object Config extends Logging { /* CONFIG Base: converted from json */ // convert Json into Config - val confBase = Config("")//Convert.jsonToClass[Config](json2, uri.toString()) + val confBase = Convert.jsonToClass[Config](json2, uri.toString()) /* CONFIG 0: apply values from package config */ // apply values from package config if need be diff --git a/src/main/scala/io/viash/config/Status.scala b/src/main/scala/io/viash/config/Status.scala index f256681fc..bb2d1d9b3 100644 --- a/src/main/scala/io/viash/config/Status.scala +++ b/src/main/scala/io/viash/config/Status.scala @@ -17,9 +17,10 @@ package io.viash.config -object Status extends Enumeration { - type Status = Value - val Enabled = Value("enabled") - val Disabled = Value("disabled") - val Deprecated = Value("deprecated") -} +enum Status: + case Enabled, Disabled, Deprecated + // type Status = Value + // val Enabled = Value("enabled") + // val Disabled = Value("disabled") + // val Deprecated = Value("deprecated") + diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index d1a8cbef2..7abe7672e 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -29,27 +29,28 @@ package object dependencies { import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // encoders and decoders for Argument - // implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict - // implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict - // implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict - // implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoderStrict - // implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoderStrict + implicit val encodeEitherStringRepository: Encoder[Either[String, Repository]] = deriveConfiguredEncoderStrict + implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict + implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict + implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict + implicit val encodeViashhubRepository: Encoder.AsObject[ViashhubRepository] = deriveConfiguredEncoderStrict + implicit val encodeLocalRepository: Encoder.AsObject[LocalRepository] = deriveConfiguredEncoderStrict // Repositories _WithName are also of type Repository, so we must define an encoder for them as well - // implicit def encodeRepository[A <: Repository]: Encoder[A] = Encoder.instance { - // par => - // val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - // val objJson = par match { - // case s: GitRepository => encodeGitRepository(s) - // case s: GithubRepository => encodeGithubRepository(s) - // case s: ViashhubRepository => encodeViashhubRepository(s) - // case s: LocalRepository => encodeLocalRepository(s) - // case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) - // case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) - // case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) - // case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) - // } - // objJson deepMerge typeJson - // } + implicit def encodeRepository[A <: Repository]: Encoder[A] = Encoder.instance { + par => + val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + val objJson = par match { + case s: GitRepository => encodeGitRepository(s) + case s: GithubRepository => encodeGithubRepository(s) + case s: ViashhubRepository => encodeViashhubRepository(s) + case s: LocalRepository => encodeLocalRepository(s) + case s: GitRepositoryWithName => encodeGitRepositoryWithName(s) + case s: GithubRepositoryWithName => encodeGithubRepositoryWithName(s) + case s: ViashhubRepositoryWithName => encodeViashhubRepositoryWithName(s) + case s: LocalRepositoryWithName => encodeLocalRepositoryWithName(s) + } + objJson deepMerge typeJson + } implicit val encodeGitRepositoryWithName: Encoder.AsObject[GitRepositoryWithName] = deriveConfiguredEncoderStrict implicit val encodeGithubRepositoryWithName: Encoder.AsObject[GithubRepositoryWithName] = deriveConfiguredEncoderStrict @@ -67,27 +68,27 @@ package object dependencies { objJson deepMerge typeJson } + implicit val decodeEitherStringRepository: Decoder[Either[String, Repository]] = deriveConfiguredDecoderFullChecks + implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks + implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks + implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks + implicit val decodeViashhubRepository: Decoder[ViashhubRepository] = deriveConfiguredDecoderFullChecks + implicit val decodeLocalRepository: Decoder[LocalRepository] = deriveConfiguredDecoderFullChecks + implicit def decodeRepository: Decoder[Repository] = Decoder.instance { + cursor => + val decoder: Decoder[Repository] = + cursor.downField("type").as[String] match { + case Right("git") => decodeGitRepository.widen + case Right("github") => decodeGithubRepository.widen + case Right("vsh") => decodeViashhubRepository.widen + case Right("local") => decodeLocalRepository.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepository](typ, List("git", "github", "vsh", "local")).widen + case Left(exception) => throw exception + } - // implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks - // implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks - // implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks - // implicit val decodeViashhubRepository: Decoder[ViashhubRepository] = deriveConfiguredDecoderFullChecks - // implicit val decodeLocalRepository: Decoder[LocalRepository] = deriveConfiguredDecoderFullChecks - // implicit def decodeRepository: Decoder[Repository] = Decoder.instance { - // cursor => - // val decoder: Decoder[Repository] = - // cursor.downField("type").as[String] match { - // case Right("git") => decodeGitRepository.widen - // case Right("github") => decodeGithubRepository.widen - // case Right("vsh") => decodeViashhubRepository.widen - // case Right("local") => decodeLocalRepository.widen - // case Right(typ) => - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[LocalRepository](typ, List("git", "github", "vsh", "local")).widen - // case Left(exception) => throw exception - // } - - // decoder(cursor) - // } + decoder(cursor) + } implicit val decodeGitRepositoryWithName: Decoder[GitRepositoryWithName] = deriveConfiguredDecoderFullChecks implicit val decodeGithubRepositoryWithName: Decoder[GithubRepositoryWithName] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index af5dfa11b..68cfdfadd 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -44,8 +44,15 @@ package object config { // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.config.resources.{decodeResource, encodeResource} + import io.viash.config.dependencies.{decodeDependency, encodeDependency} + import io.viash.config.dependencies.{decodeRepositoryWithName, encodeRepositoryWithName} + import io.viash.runners.{decodeRunner, encodeRunner} + import io.viash.engines.{decodeEngine, encodeEngine} + import io.viash.packageConfig.{decodePackageConfig, encodePackageConfig} + // encoders and decoders for Config - // implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] + implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] // implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config] /*.prepare{ checkDeprecation[Config](_) @@ -273,23 +280,26 @@ package object config { "Could not convert json to Config." ) }*/ + implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderFullChecks - // implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder - // implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks + implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder + implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks // encoder and decoder for Author implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks // encoder and decoder for Requirements - // implicit val encodeComputationalRequirements: Encoder.AsObject[ComputationalRequirements] = deriveConfiguredEncoder - // implicit val decodeComputationalRequirements: Decoder[ComputationalRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeComputationalRequirements: Encoder.AsObject[ComputationalRequirements] = deriveConfiguredEncoder + implicit val decodeComputationalRequirements: Decoder[ComputationalRequirements] = deriveConfiguredDecoderFullChecks // encoder and decoder for ArgumentGroup - // implicit val encodeArgumentGroup: Encoder.AsObject[ArgumentGroup] = deriveConfiguredEncoder - // implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks + implicit val encodeArgumentGroup: Encoder.AsObject[ArgumentGroup] = deriveConfiguredEncoder + implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks // encoder and decoder for Status, make string lowercase before decoding + implicit val encodeStatus: Encoder[Status] = deriveConfiguredEncoder + implicit val decodeStatus: Decoder[Status] = deriveConfiguredDecoderFullChecks // implicit val encodeStatus: Encoder[Status] = Encoder.encodeEnumeration(Status) // implicit val decodeStatus: Decoder[Status] = Decoder.decodeEnumeration(Status).prepare { // _.withFocus(_.mapString(_.toLowerCase())) diff --git a/src/main/scala/io/viash/config/resources/package.scala b/src/main/scala/io/viash/config/resources/package.scala index 030d61198..3257a7657 100644 --- a/src/main/scala/io/viash/config/resources/package.scala +++ b/src/main/scala/io/viash/config/resources/package.scala @@ -26,9 +26,9 @@ import io.circe.ACursor package object resources { - // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // import io.viash.helpers.circe._ - // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe._ + import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // implicit val encodeURI: Encoder[URI] = Encoder.instance { // uri => Json.fromString(uri.toString) @@ -37,78 +37,78 @@ package object resources { // cursor => cursor.value.as[String].map(new URI(_)) // } - // // encoders and decoders for Object - // implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoderStrict[BashScript] - // implicit val encodePythonScript: Encoder.AsObject[PythonScript] = deriveConfiguredEncoderStrict[PythonScript] - // implicit val encodeRScript: Encoder.AsObject[RScript] = deriveConfiguredEncoderStrict[RScript] - // implicit val encodeJavaScriptScript: Encoder.AsObject[JavaScriptScript] = deriveConfiguredEncoderStrict[JavaScriptScript] - // implicit val encodeNextflowScript: Encoder.AsObject[NextflowScript] = deriveConfiguredEncoderStrict[NextflowScript] - // implicit val encodeScalaScript: Encoder.AsObject[ScalaScript] = deriveConfiguredEncoderStrict[ScalaScript] - // implicit val encodeCSharpScript: Encoder.AsObject[CSharpScript] = deriveConfiguredEncoderStrict[CSharpScript] - // implicit val encodeExecutable: Encoder.AsObject[Executable] = deriveConfiguredEncoderStrict[Executable] - // implicit val encodePlainFile: Encoder.AsObject[PlainFile] = deriveConfiguredEncoderStrict[PlainFile] + // encoders and decoders for Object + implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoderStrict[BashScript] + implicit val encodePythonScript: Encoder.AsObject[PythonScript] = deriveConfiguredEncoderStrict[PythonScript] + implicit val encodeRScript: Encoder.AsObject[RScript] = deriveConfiguredEncoderStrict[RScript] + implicit val encodeJavaScriptScript: Encoder.AsObject[JavaScriptScript] = deriveConfiguredEncoderStrict[JavaScriptScript] + implicit val encodeNextflowScript: Encoder.AsObject[NextflowScript] = deriveConfiguredEncoderStrict[NextflowScript] + implicit val encodeScalaScript: Encoder.AsObject[ScalaScript] = deriveConfiguredEncoderStrict[ScalaScript] + implicit val encodeCSharpScript: Encoder.AsObject[CSharpScript] = deriveConfiguredEncoderStrict[CSharpScript] + implicit val encodeExecutable: Encoder.AsObject[Executable] = deriveConfiguredEncoderStrict[Executable] + implicit val encodePlainFile: Encoder.AsObject[PlainFile] = deriveConfiguredEncoderStrict[PlainFile] - // implicit def encodeResource[A <: Resource]: Encoder[A] = Encoder.instance { - // par => - // val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - // val objJson = par match { - // case s: BashScript => encodeBashScript(s) - // case s: PythonScript => encodePythonScript(s) - // case s: RScript => encodeRScript(s) - // case s: JavaScriptScript => encodeJavaScriptScript(s) - // case s: NextflowScript => encodeNextflowScript(s) - // case s: ScalaScript => encodeScalaScript(s) - // case s: CSharpScript => encodeCSharpScript(s) - // case s: Executable => encodeExecutable(s) - // case s: PlainFile => encodePlainFile(s) - // } - // objJson deepMerge typeJson - // } + implicit def encodeResource[A <: Resource]: Encoder[A] = Encoder.instance { + par => + val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) + val objJson = par match { + case s: BashScript => encodeBashScript(s) + case s: PythonScript => encodePythonScript(s) + case s: RScript => encodeRScript(s) + case s: JavaScriptScript => encodeJavaScriptScript(s) + case s: NextflowScript => encodeNextflowScript(s) + case s: ScalaScript => encodeScalaScript(s) + case s: CSharpScript => encodeCSharpScript(s) + case s: Executable => encodeExecutable(s) + case s: PlainFile => encodePlainFile(s) + } + objJson deepMerge typeJson + } - // val setDestToPathOrDefault = (default: String) => (aCursor: ACursor) => {aCursor.withFocus(js => { - // js.mapObject{ obj => - // // when json defines 'text' but no 'dest' set - // // if has 'path' -> switch 'path' to 'dest' - // // else if no 'path' or 'dest' -> set 'dest' to default value - // if (obj.contains("text") && !obj.contains("dest")) { - // if (obj.contains("path")) - // obj.add("dest", obj("path").get).remove("path") - // else - // obj.add("dest", Json.fromString(default)) - // } else { - // obj - // } - // } - // })} + val setDestToPathOrDefault = (default: String) => (aCursor: ACursor) => {aCursor.withFocus(js => { + js.mapObject{ obj => + // when json defines 'text' but no 'dest' set + // if has 'path' -> switch 'path' to 'dest' + // else if no 'path' or 'dest' -> set 'dest' to default value + if (obj.contains("text") && !obj.contains("dest")) { + if (obj.contains("path")) + obj.add("dest", obj("path").get).remove("path") + else + obj.add("dest", Json.fromString(default)) + } else { + obj + } + } + })} - // implicit val decodeBashScript: Decoder[BashScript] = deriveConfiguredDecoderFullChecks[BashScript].prepare { setDestToPathOrDefault("./script.sh") } - // implicit val decodePythonScript: Decoder[PythonScript] = deriveConfiguredDecoderFullChecks[PythonScript].prepare { setDestToPathOrDefault("./script.py") } - // implicit val decodeRScript: Decoder[RScript] = deriveConfiguredDecoderFullChecks[RScript].prepare { setDestToPathOrDefault("./script.R") } - // implicit val decodeJavaScriptScript: Decoder[JavaScriptScript] = deriveConfiguredDecoderFullChecks[JavaScriptScript].prepare { setDestToPathOrDefault("./script.js") } - // implicit val decodeNextflowScript: Decoder[NextflowScript] = deriveConfiguredDecoderFullChecks[NextflowScript].prepare { setDestToPathOrDefault("./script.nf") } - // implicit val decodeScalaScript: Decoder[ScalaScript] = deriveConfiguredDecoderFullChecks[ScalaScript].prepare { setDestToPathOrDefault("./script.scala") } - // implicit val decodeCSharpScript: Decoder[CSharpScript] = deriveConfiguredDecoderFullChecks[CSharpScript].prepare { setDestToPathOrDefault("./script.csx") } - // implicit val decodeExecutable: Decoder[Executable] = deriveConfiguredDecoderFullChecks - // implicit val decodePlainFile: Decoder[PlainFile] = deriveConfiguredDecoderFullChecks[PlainFile].prepare { setDestToPathOrDefault("./text.txt") } + implicit val decodeBashScript: Decoder[BashScript] = deriveConfiguredDecoderFullChecks[BashScript].prepare { setDestToPathOrDefault("./script.sh") } + implicit val decodePythonScript: Decoder[PythonScript] = deriveConfiguredDecoderFullChecks[PythonScript].prepare { setDestToPathOrDefault("./script.py") } + implicit val decodeRScript: Decoder[RScript] = deriveConfiguredDecoderFullChecks[RScript].prepare { setDestToPathOrDefault("./script.R") } + implicit val decodeJavaScriptScript: Decoder[JavaScriptScript] = deriveConfiguredDecoderFullChecks[JavaScriptScript].prepare { setDestToPathOrDefault("./script.js") } + implicit val decodeNextflowScript: Decoder[NextflowScript] = deriveConfiguredDecoderFullChecks[NextflowScript].prepare { setDestToPathOrDefault("./script.nf") } + implicit val decodeScalaScript: Decoder[ScalaScript] = deriveConfiguredDecoderFullChecks[ScalaScript].prepare { setDestToPathOrDefault("./script.scala") } + implicit val decodeCSharpScript: Decoder[CSharpScript] = deriveConfiguredDecoderFullChecks[CSharpScript].prepare { setDestToPathOrDefault("./script.csx") } + implicit val decodeExecutable: Decoder[Executable] = deriveConfiguredDecoderFullChecks + implicit val decodePlainFile: Decoder[PlainFile] = deriveConfiguredDecoderFullChecks[PlainFile].prepare { setDestToPathOrDefault("./text.txt") } - // implicit def decodeResource: Decoder[Resource] = Decoder.instance { - // cursor => - // val decoder: Decoder[Resource] = - // cursor.downField("type").as[String] match { - // case Right("bash_script") => decodeBashScript.widen - // case Right("python_script") => decodePythonScript.widen - // case Right("r_script") => decodeRScript.widen - // case Right("javascript_script") => decodeJavaScriptScript.widen - // case Right("nextflow_script") => decodeNextflowScript.widen - // case Right("scala_script") => decodeScalaScript.widen - // case Right("csharp_script") => decodeCSharpScript.widen - // case Right("executable") => decodeExecutable.widen - // case Right("file") => decodePlainFile.widen - // case Right(typ) => - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, Script.companions.map(c => c.`type`) ++ List("executable", "file")).widen - // case Left(_) => decodePlainFile.widen // default is a simple file - // } + implicit def decodeResource: Decoder[Resource] = Decoder.instance { + cursor => + val decoder: Decoder[Resource] = + cursor.downField("type").as[String] match { + case Right("bash_script") => decodeBashScript.widen + case Right("python_script") => decodePythonScript.widen + case Right("r_script") => decodeRScript.widen + case Right("javascript_script") => decodeJavaScriptScript.widen + case Right("nextflow_script") => decodeNextflowScript.widen + case Right("scala_script") => decodeScalaScript.widen + case Right("csharp_script") => decodeCSharpScript.widen + case Right("executable") => decodeExecutable.widen + case Right("file") => decodePlainFile.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, Script.companions.map(c => c.`type`) ++ List("executable", "file")).widen + case Left(_) => decodePlainFile.widen // default is a simple file + } - // decoder(cursor) - // } + decoder(cursor) + } } \ No newline at end of file diff --git a/src/main/scala/io/viash/engines/docker/package.scala b/src/main/scala/io/viash/engines/docker/package.scala index ced79ff05..0a732628c 100644 --- a/src/main/scala/io/viash/engines/docker/package.scala +++ b/src/main/scala/io/viash/engines/docker/package.scala @@ -24,18 +24,18 @@ package object docker { import io.viash.helpers.circe._ // encoder and decoder for resolvevolume - // implicit val encodeResolveVolume: Encoder[DockerResolveVolume] = Encoder.instance { - // v => Json.fromString(v.toString) - // } - // implicit val decodeResolveVolume: Decoder[DockerResolveVolume] = Decoder.instance { - // cursor => - // cursor.value.as[String].map(s => - // s.toLowerCase() match { - // case "manual" => Manual - // case "auto" | "automatic" => Automatic - // } - // ) - // } + implicit val encodeResolveVolume: Encoder[DockerResolveVolume] = Encoder.instance { + v => Json.fromString(v.toString) + } + implicit val decodeResolveVolume: Decoder[DockerResolveVolume] = Decoder.instance { + cursor => + cursor.value.as[String].map(s => + s.toLowerCase() match { + case "manual" => Manual + case "auto" | "automatic" => Automatic + } + ) + } } diff --git a/src/main/scala/io/viash/engines/package.scala b/src/main/scala/io/viash/engines/package.scala index 1ad03d84e..f61995ed4 100644 --- a/src/main/scala/io/viash/engines/package.scala +++ b/src/main/scala/io/viash/engines/package.scala @@ -25,34 +25,37 @@ import cats.syntax.functor._ package object engines { import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - - // implicit val encodeDockerEngine: Encoder.AsObject[DockerEngine] = deriveConfiguredEncoder - // implicit val decodeDockerEngine: Decoder[DockerEngine] = deriveConfiguredDecoderFullChecks - - // implicit val encodeNativeEngine: Encoder.AsObject[NativeEngine] = deriveConfiguredEncoder - // implicit val decodeNativeEngine: Decoder[NativeEngine] = deriveConfiguredDecoderFullChecks - - // implicit def encodeEngine[A <: Engine]: Encoder[A] = Encoder.instance { - // engine => - // val typeJson = Json.obj("type" -> Json.fromString(engine.`type`)) - // val objJson = engine match { - // case s: DockerEngine => encodeDockerEngine(s) - // case s: NativeEngine => encodeNativeEngine(s) - // } - // objJson deepMerge typeJson - // } - - // implicit def decodeEngine: Decoder[Engine] = Decoder.instance { - // cursor => - // val decoder: Decoder[Engine] = - // cursor.downField("type").as[String] match { - // case Right("docker") => decodeDockerEngine.widen - // case Right("native") => decodeNativeEngine.widen - // case Right(typ) => - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativeEngine](typ, List("docker", "native")).widen - // case Left(exception) => throw exception - // } - - // decoder(cursor) - // } + import io.viash.helpers.circe.DeriveConfiguredEncoder._ + + import io.viash.engines.requirements.{decodeRequirements, encodeRequirements} + + implicit val encodeDockerEngine: Encoder.AsObject[DockerEngine] = deriveConfiguredEncoder + implicit val decodeDockerEngine: Decoder[DockerEngine] = deriveConfiguredDecoderFullChecks + + implicit val encodeNativeEngine: Encoder.AsObject[NativeEngine] = deriveConfiguredEncoder + implicit val decodeNativeEngine: Decoder[NativeEngine] = deriveConfiguredDecoderFullChecks + + implicit def encodeEngine[A <: Engine]: Encoder[A] = Encoder.instance { + engine => + val typeJson = Json.obj("type" -> Json.fromString(engine.`type`)) + val objJson = engine match { + case s: DockerEngine => encodeDockerEngine(s) + case s: NativeEngine => encodeNativeEngine(s) + } + objJson deepMerge typeJson + } + + implicit def decodeEngine: Decoder[Engine] = Decoder.instance { + cursor => + val decoder: Decoder[Engine] = + cursor.downField("type").as[String] match { + case Right("docker") => decodeDockerEngine.widen + case Right("native") => decodeNativeEngine.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativeEngine](typ, List("docker", "native")).widen + case Left(exception) => throw exception + } + + decoder(cursor) + } } diff --git a/src/main/scala/io/viash/engines/requirements/package.scala b/src/main/scala/io/viash/engines/requirements/package.scala index 9b67f4246..554070f8a 100644 --- a/src/main/scala/io/viash/engines/requirements/package.scala +++ b/src/main/scala/io/viash/engines/requirements/package.scala @@ -23,63 +23,63 @@ import cats.syntax.functor._ // for .widen package object requirements { import io.viash.helpers.circe._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredEncoder._ - // implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder - // implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder + implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodePythonRequirements: Encoder.AsObject[PythonRequirements] = deriveConfiguredEncoder - // implicit val decodePythonRequirements: Decoder[PythonRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodePythonRequirements: Encoder.AsObject[PythonRequirements] = deriveConfiguredEncoder + implicit val decodePythonRequirements: Decoder[PythonRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodeRubyRequirements: Encoder.AsObject[RubyRequirements] = deriveConfiguredEncoder - // implicit val decodeRubyRequirements: Decoder[RubyRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeRubyRequirements: Encoder.AsObject[RubyRequirements] = deriveConfiguredEncoder + implicit val decodeRubyRequirements: Decoder[RubyRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodeJavaScriptRequirements: Encoder.AsObject[JavaScriptRequirements] = deriveConfiguredEncoder - // implicit val decodeJavaScriptRequirements: Decoder[JavaScriptRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeJavaScriptRequirements: Encoder.AsObject[JavaScriptRequirements] = deriveConfiguredEncoder + implicit val decodeJavaScriptRequirements: Decoder[JavaScriptRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodeAptRequirements: Encoder.AsObject[AptRequirements] = deriveConfiguredEncoder - // implicit val decodeAptRequirements: Decoder[AptRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeAptRequirements: Encoder.AsObject[AptRequirements] = deriveConfiguredEncoder + implicit val decodeAptRequirements: Decoder[AptRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodeYumRequirements: Encoder.AsObject[YumRequirements] = deriveConfiguredEncoder - // implicit val decodeYumRequirements: Decoder[YumRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeYumRequirements: Encoder.AsObject[YumRequirements] = deriveConfiguredEncoder + implicit val decodeYumRequirements: Decoder[YumRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodeApkRequirements: Encoder.AsObject[ApkRequirements] = deriveConfiguredEncoder - // implicit val decodeApkRequirements: Decoder[ApkRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeApkRequirements: Encoder.AsObject[ApkRequirements] = deriveConfiguredEncoder + implicit val decodeApkRequirements: Decoder[ApkRequirements] = deriveConfiguredDecoderFullChecks - // implicit val encodeDockerRequirements: Encoder.AsObject[DockerRequirements] = deriveConfiguredEncoder - // implicit val decodeDockerRequirements: Decoder[DockerRequirements] = deriveConfiguredDecoderFullChecks + implicit val encodeDockerRequirements: Encoder.AsObject[DockerRequirements] = deriveConfiguredEncoder + implicit val decodeDockerRequirements: Decoder[DockerRequirements] = deriveConfiguredDecoderFullChecks implicit def encodeRequirements[A <: Requirements]: Encoder[A] = Encoder.instance { reqs => val typeJson = Json.obj("type" -> Json.fromString(reqs.`type`)) - // val objJson = reqs match { - // case s: ApkRequirements => encodeApkRequirements(s) - // case s: AptRequirements => encodeAptRequirements(s) - // case s: YumRequirements => encodeYumRequirements(s) - // case s: DockerRequirements => encodeDockerRequirements(s) - // case s: PythonRequirements => encodePythonRequirements(s) - // case s: RRequirements => encodeRRequirements(s) - // case s: JavaScriptRequirements => encodeJavaScriptRequirements(s) - // case s: RubyRequirements => encodeRubyRequirements(s) - // } - // objJson deepMerge typeJson - typeJson + val objJson = reqs match { + case s: ApkRequirements => encodeApkRequirements(s) + case s: AptRequirements => encodeAptRequirements(s) + case s: YumRequirements => encodeYumRequirements(s) + case s: DockerRequirements => encodeDockerRequirements(s) + case s: PythonRequirements => encodePythonRequirements(s) + case s: RRequirements => encodeRRequirements(s) + case s: JavaScriptRequirements => encodeJavaScriptRequirements(s) + case s: RubyRequirements => encodeRubyRequirements(s) + } + objJson deepMerge typeJson } implicit def decodeRequirements: Decoder[Requirements] = Decoder.instance { cursor => val decoder: Decoder[Requirements] = cursor.downField("type").as[String] match { - // case Right("apk") => decodeApkRequirements.widen - // case Right("apt") => decodeAptRequirements.widen - // case Right("yum") => decodeYumRequirements.widen - // case Right("docker") => decodeDockerRequirements.widen - // case Right("python") => decodePythonRequirements.widen - // case Right("r") => decodeRRequirements.widen - // case Right("javascript") => decodeJavaScriptRequirements.widen - // case Right("ruby") => decodeRubyRequirements.widen - // case Right(typ) => - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ApkRequirements](typ, List("apk", "apt", "yum", "docker", "python", "r", "javascript", "ruby")).widen + case Right("apk") => decodeApkRequirements.widen + case Right("apt") => decodeAptRequirements.widen + case Right("yum") => decodeYumRequirements.widen + case Right("docker") => decodeDockerRequirements.widen + case Right("python") => decodePythonRequirements.widen + case Right("r") => decodeRRequirements.widen + case Right("javascript") => decodeJavaScriptRequirements.widen + case Right("ruby") => decodeRubyRequirements.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ApkRequirements](typ, List("apk", "apt", "yum", "docker", "python", "r", "javascript", "ruby")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/functionality/package.scala b/src/main/scala/io/viash/functionality/package.scala index 33d542e17..dd1b91338 100644 --- a/src/main/scala/io/viash/functionality/package.scala +++ b/src/main/scala/io/viash/functionality/package.scala @@ -36,13 +36,17 @@ import config.arguments._ package object functionality extends Logging { // import implicits import io.viash.helpers.circe._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - // import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + + import io.viash.config.{decodeAuthor, decodeArgumentGroup, decodeStatus, decodeComputationalRequirements, decodeReferences, decodeLinks} + import io.viash.config.resources.decodeResource + import io.viash.config.dependencies.{decodeDependency, decodeRepositoryWithName} // encoder and decoder for Functionality // implicit val encodeFunctionality: Encoder.AsObject[Functionality] = deriveConfiguredEncoderStrict[Functionality] - // implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderFullChecks + implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/platforms/package.scala b/src/main/scala/io/viash/platforms/package.scala index 5944be862..23ae7231b 100644 --- a/src/main/scala/io/viash/platforms/package.scala +++ b/src/main/scala/io/viash/platforms/package.scala @@ -25,40 +25,48 @@ import cats.syntax.functor._ // for .widen package object platforms { import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredEncoder._ - // implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder - // implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks + import io.viash.runners.nextflow.{decodeNextflowDirectives, encodeNextflowDirectives} + import io.viash.engines.docker.{decodeResolveVolume, encodeResolveVolume} + import io.viash.runners.executable.{decodeSetupStrategy, encodeSetupStrategy} + import io.viash.engines.requirements.{decodeRequirements, encodeRequirements} + import io.viash.runners.nextflow.{decodeNextflowAuto, encodeNextflowAuto} + import io.viash.runners.nextflow.{decodeNextflowConfig, encodeNextflowConfig} - // implicit val encodeNextflowPlatform: Encoder.AsObject[NextflowPlatform] = deriveConfiguredEncoder - // implicit val decodeNextflowPlatform: Decoder[NextflowPlatform] = deriveConfiguredDecoderFullChecks + implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder + implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks - // implicit val encodeNativePlatform: Encoder.AsObject[NativePlatform] = deriveConfiguredEncoder - // implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderFullChecks + implicit val encodeNextflowPlatform: Encoder.AsObject[NextflowPlatform] = deriveConfiguredEncoder + implicit val decodeNextflowPlatform: Decoder[NextflowPlatform] = deriveConfiguredDecoderFullChecks - // implicit def encodePlatform[A <: Platform]: Encoder[A] = Encoder.instance { - // platform => - // val typeJson = Json.obj("type" -> Json.fromString(platform.`type`)) - // val objJson = platform match { - // case s: DockerPlatform => encodeDockerPlatform(s) - // case s: NextflowPlatform => encodeNextflowPlatform(s) - // case s: NativePlatform => encodeNativePlatform(s) - // } - // objJson deepMerge typeJson - // } + implicit val encodeNativePlatform: Encoder.AsObject[NativePlatform] = deriveConfiguredEncoder + implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderFullChecks - // implicit def decodePlatform: Decoder[Platform] = Decoder.instance { - // cursor => - // val decoder: Decoder[Platform] = - // cursor.downField("type").as[String] match { - // case Right("docker") => decodeDockerPlatform.widen - // case Right("native") => decodeNativePlatform.widen - // case Right("nextflow") => decodeNextflowPlatform.widen - // case Right(typ) => - // //throw new RuntimeException("Type " + typ + " is not recognised.") - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativePlatform](typ, List("docker", "native", "nextflow")).widen - // case Left(exception) => throw exception - // } + implicit def encodePlatform[A <: Platform]: Encoder[A] = Encoder.instance { + platform => + val typeJson = Json.obj("type" -> Json.fromString(platform.`type`)) + val objJson = platform match { + case s: DockerPlatform => encodeDockerPlatform(s) + case s: NextflowPlatform => encodeNextflowPlatform(s) + case s: NativePlatform => encodeNativePlatform(s) + } + objJson deepMerge typeJson + } - // decoder(cursor) - // } + implicit def decodePlatform: Decoder[Platform] = Decoder.instance { + cursor => + val decoder: Decoder[Platform] = + cursor.downField("type").as[String] match { + case Right("docker") => decodeDockerPlatform.widen + case Right("native") => decodeNativePlatform.widen + case Right("nextflow") => decodeNextflowPlatform.widen + case Right(typ) => + //throw new RuntimeException("Type " + typ + " is not recognised.") + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativePlatform](typ, List("docker", "native", "nextflow")).widen + case Left(exception) => throw exception + } + + decoder(cursor) + } } diff --git a/src/main/scala/io/viash/runners/executable/package.scala b/src/main/scala/io/viash/runners/executable/package.scala index 8ee1de674..f21dfd99a 100644 --- a/src/main/scala/io/viash/runners/executable/package.scala +++ b/src/main/scala/io/viash/runners/executable/package.scala @@ -24,18 +24,18 @@ package object executable { import io.viash.helpers.circe._ // encoder and decoder for setup strategy - // implicit val encodeSetupStrategy: Encoder[DockerSetupStrategy] = Encoder.instance { - // dss => Json.fromString(dss.id.toLowerCase) - // } - // implicit val decodeSetupStrategy: Decoder[DockerSetupStrategy] = Decoder.instance { - // cursor => - // cursor.value.as[String].map { s => - // val id = s.toLowerCase.replaceAll("_", "") - // DockerSetupStrategy.map.applyOrElse(id, - // (key: String) => throw new Exception(s"Docker Setup Strategy '$key' not found.") - // ) - // } - // } + implicit val encodeSetupStrategy: Encoder[DockerSetupStrategy] = Encoder.instance { + dss => Json.fromString(dss.id.toLowerCase) + } + implicit val decodeSetupStrategy: Decoder[DockerSetupStrategy] = Decoder.instance { + cursor => + cursor.value.as[String].map { s => + val id = s.toLowerCase.replaceAll("_", "") + DockerSetupStrategy.map.applyOrElse(id, + (key: String) => throw new Exception(s"Docker Setup Strategy '$key' not found.") + ) + } + } } diff --git a/src/main/scala/io/viash/runners/nextflow/package.scala b/src/main/scala/io/viash/runners/nextflow/package.scala index 2ca66a11d..232127405 100644 --- a/src/main/scala/io/viash/runners/nextflow/package.scala +++ b/src/main/scala/io/viash/runners/nextflow/package.scala @@ -18,18 +18,32 @@ package io.viash.runners import io.circe.{Decoder, Encoder, Json} +import io.circe.derivation.ConfiguredDecoder // import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object nextflow { import io.viash.helpers.circe._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ + import io.viash.helpers.circe.DeriveConfiguredEncoder._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder - // implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks + // implicit val encodeEitherBooleanString: Encoder[Either[Boolean, String]] = Encoder.derived + // implicit val decodeEitherBooleanString: Decoder[Either[Boolean, String]] = Decoder.derived + // implicit val encodeEitherMapStringStringString: Encoder[Either[Map[String, String], String]] = Encoder.derived + // implicit val decodeEitherMapStringStringString: Decoder[Either[Map[String, String], String]] = Decoder.derived + // implicit val encodeEitherIntString: Encoder[Either[Int, String]] = Encoder.derived + // implicit val decodeEitherIntString: Decoder[Either[Int, String]] = Decoder.derived + // implicit val encodeEitherStringInt: Encoder[Either[String, Int]] = Encoder.derived + // implicit val decodeEitherStringInt: Decoder[Either[String, Int]] = Decoder.derived + // implicit val encodeEitherStringMapStringString: Encoder[Either[String, Map[String, String]]] = Encoder.derived + // implicit val decodeEitherStringMapStringString: Decoder[Either[String, Map[String, String]]] = Decoder.derived - // implicit val encodeNextflowAuto: Encoder.AsObject[NextflowAuto] = deriveConfiguredEncoder - // implicit val decodeNextflowAuto: Decoder[NextflowAuto] = deriveConfiguredDecoderFullChecks - // implicit val encodeNextflowConfig: Encoder.AsObject[NextflowConfig] = deriveConfiguredEncoder - // implicit val decodeNextflowConfig: Decoder[NextflowConfig] = deriveConfiguredDecoderFullChecks + implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder + implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks + + implicit val encodeNextflowAuto: Encoder.AsObject[NextflowAuto] = deriveConfiguredEncoder + implicit val decodeNextflowAuto: Decoder[NextflowAuto] = deriveConfiguredDecoderFullChecks + + implicit val encodeNextflowConfig: Encoder.AsObject[NextflowConfig] = deriveConfiguredEncoder + implicit val decodeNextflowConfig: Decoder[NextflowConfig] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/runners/package.scala b/src/main/scala/io/viash/runners/package.scala index 8f64f12c1..a9925895e 100644 --- a/src/main/scala/io/viash/runners/package.scala +++ b/src/main/scala/io/viash/runners/package.scala @@ -21,38 +21,44 @@ import io.circe.{Decoder, Encoder, Json} // import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import cats.syntax.functor._ // for .widen +import io.viash.helpers.circe.DeriveConfiguredEncoder.deriveConfiguredEncoder package object runners { import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = deriveEncoder - // implicit val decodeExecutableRunner: Decoder[ExecutableRunner] = deriveConfiguredDecoderFullChecks - - // implicit val encodeNextflowRunner: Encoder.AsObject[NextflowRunner] = deriveEncoder - // implicit val decodeNextflowRunner: Decoder[NextflowRunner] = deriveConfiguredDecoderFullChecks - - // implicit def encodeRunner[A <: Runner]: Encoder[A] = Encoder.instance { - // runner => - // val typeJson = Json.obj("type" -> Json.fromString(runner.`type`)) - // val objJson = runner match { - // case s: ExecutableRunner => encodeExecutableRunner(s) - // case s: NextflowRunner => encodeNextflowRunner(s) - // } - // objJson deepMerge typeJson - // } - - // implicit def decodeRunner: Decoder[Runner] = Decoder.instance { - // cursor => - // val decoder: Decoder[Runner] = - // cursor.downField("type").as[String] match { - // case Right("executable") => decodeExecutableRunner.widen - // case Right("nextflow") => decodeNextflowRunner.widen - // case Right(typ) => - // DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ExecutableRunner](typ, List("executable", "nextflow")).widen - // case Left(exception) => throw exception - // } - - // decoder(cursor) - // } + import io.viash.runners.executable.{decodeSetupStrategy, encodeSetupStrategy} + import io.viash.runners.nextflow.{decodeNextflowDirectives, encodeNextflowDirectives} + import io.viash.runners.nextflow.{decodeNextflowAuto, encodeNextflowAuto} + import io.viash.runners.nextflow.{decodeNextflowConfig, encodeNextflowConfig} + + implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = Encoder.derived + implicit val decodeExecutableRunner: Decoder[ExecutableRunner] = deriveConfiguredDecoderFullChecks + + implicit val encodeNextflowRunner: Encoder.AsObject[NextflowRunner] = Encoder.derived + implicit val decodeNextflowRunner: Decoder[NextflowRunner] = deriveConfiguredDecoderFullChecks + + implicit def encodeRunner[A <: Runner]: Encoder[A] = Encoder.instance { + runner => + val typeJson = Json.obj("type" -> Json.fromString(runner.`type`)) + val objJson = runner match { + case s: ExecutableRunner => encodeExecutableRunner(s) + case s: NextflowRunner => encodeNextflowRunner(s) + } + objJson deepMerge typeJson + } + + implicit def decodeRunner: Decoder[Runner] = Decoder.instance { + cursor => + val decoder: Decoder[Runner] = + cursor.downField("type").as[String] match { + case Right("executable") => decodeExecutableRunner.widen + case Right("nextflow") => decodeNextflowRunner.widen + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ExecutableRunner](typ, List("executable", "nextflow")).widen + case Left(exception) => throw exception + } + + decoder(cursor) + } } From b186934dae475db400932a61a4c5cc9436355a19 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 29 Aug 2024 17:14:18 +0200 Subject: [PATCH 04/46] fudge BuildStatus type to assist type matching, set max-inlines in sbt, enable either decoder/encoder back again I thought either was supported in circe now, need to double check --- build.sbt | 1 + src/main/scala/io/viash/ViashBuild.scala | 3 ++- src/main/scala/io/viash/helpers/circe/package.scala | 12 ++++++------ .../scala/io/viash/runners/nextflow/package.scala | 13 ------------- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/build.sbt b/build.sbt index 4746bd264..aca98f76a 100644 --- a/build.sbt +++ b/build.sbt @@ -32,6 +32,7 @@ libraryDependencies ++= Seq( ) scalacOptions ++= Seq("-unchecked", "-deprecation", "-explain") +scalacOptions ++= Seq("-Xmax-inlines", "50") organization := "Data Intuitive" startYear := Some(2020) diff --git a/src/main/scala/io/viash/ViashBuild.scala b/src/main/scala/io/viash/ViashBuild.scala index 72b7aa032..64f277f05 100644 --- a/src/main/scala/io/viash/ViashBuild.scala +++ b/src/main/scala/io/viash/ViashBuild.scala @@ -19,6 +19,7 @@ package io.viash import java.nio.file.{Files, Paths} import scala.sys.process.{Process, ProcessLogger} +import io.viash.helpers.status import io.viash.helpers.status._ import config._ @@ -31,7 +32,7 @@ object ViashBuild extends Logging { output: String, setup: Option[String] = None, push: Boolean = false - ): Status = { + ): status.Status = { val resources = appliedConfig.generateRunner(false) // create dir diff --git a/src/main/scala/io/viash/helpers/circe/package.scala b/src/main/scala/io/viash/helpers/circe/package.scala index 46fad0733..b44730a21 100644 --- a/src/main/scala/io/viash/helpers/circe/package.scala +++ b/src/main/scala/io/viash/helpers/circe/package.scala @@ -29,13 +29,13 @@ package object circe { // // encoder and decoder for Either - // implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { - // _.fold(ea(_), eb(_)) - // } + implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { + _.fold(ea(_), eb(_)) + } - // implicit def decodeEither[A,B](implicit a: Decoder[A], b: Decoder[B]): Decoder[Either[A,B]] = { - // a either b - // } + implicit def decodeEither[A,B](implicit a: Decoder[A], b: Decoder[B]): Decoder[Either[A,B]] = { + a either b + } // encoder and decoder for OneOrMore implicit def encodeOneOrMore[A](implicit enc: Encoder[List[A]]): Encoder[OneOrMore[A]] = { diff --git a/src/main/scala/io/viash/runners/nextflow/package.scala b/src/main/scala/io/viash/runners/nextflow/package.scala index 232127405..320ab11a8 100644 --- a/src/main/scala/io/viash/runners/nextflow/package.scala +++ b/src/main/scala/io/viash/runners/nextflow/package.scala @@ -19,25 +19,12 @@ package io.viash.runners import io.circe.{Decoder, Encoder, Json} import io.circe.derivation.ConfiguredDecoder -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object nextflow { import io.viash.helpers.circe._ import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // implicit val encodeEitherBooleanString: Encoder[Either[Boolean, String]] = Encoder.derived - // implicit val decodeEitherBooleanString: Decoder[Either[Boolean, String]] = Decoder.derived - // implicit val encodeEitherMapStringStringString: Encoder[Either[Map[String, String], String]] = Encoder.derived - // implicit val decodeEitherMapStringStringString: Decoder[Either[Map[String, String], String]] = Decoder.derived - // implicit val encodeEitherIntString: Encoder[Either[Int, String]] = Encoder.derived - // implicit val decodeEitherIntString: Decoder[Either[Int, String]] = Decoder.derived - // implicit val encodeEitherStringInt: Encoder[Either[String, Int]] = Encoder.derived - // implicit val decodeEitherStringInt: Decoder[Either[String, Int]] = Decoder.derived - // implicit val encodeEitherStringMapStringString: Encoder[Either[String, Map[String, String]]] = Encoder.derived - // implicit val decodeEitherStringMapStringString: Decoder[Either[String, Map[String, String]]] = Decoder.derived - - implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks From 1b8d8ae651a58e8c9db331cb9d5bd7a4a6ae9340 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 29 Aug 2024 17:23:46 +0200 Subject: [PATCH 05/46] re-enable json encoding and decoding calls that were previously disabled --- src/main/scala/io/viash/config/Config.scala | 2 +- src/main/scala/io/viash/config/ConfigMeta.scala | 2 +- .../scala/io/viash/runners/nextflow/NextflowHelper.scala | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index e2273b030..dd96e5d6a 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -683,7 +683,7 @@ object Config extends Logging { val conf1 = if (confMods.postparseCommands.nonEmpty) { // turn config back into json - val js = Json.Null//encodeConfig(conf0) + val js = encodeConfig(conf0) // apply config mods val modifiedJs = confMods(js, preparse = false) // turn json back into a config diff --git a/src/main/scala/io/viash/config/ConfigMeta.scala b/src/main/scala/io/viash/config/ConfigMeta.scala index ffa71a2a3..63a56cedb 100644 --- a/src/main/scala/io/viash/config/ConfigMeta.scala +++ b/src/main/scala/io/viash/config/ConfigMeta.scala @@ -62,7 +62,7 @@ object ConfigMeta { )) ) - val encodedConfig: Json = Json.Null//encodeConfig(anonymizedConfig) + val encodedConfig: Json = encodeConfig(anonymizedConfig) // drop empty & null values recursively except all "info" fields val cleanEncodedConfig = encodedConfig.dropEmptyRecursivelyExcept(Seq("info", ".engines.entrypoint", ".engines.cmd")) // get config.info and *do* clean it diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala index ffb1343de..9986dee04 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala @@ -33,6 +33,8 @@ import java.nio.file.Paths import io.viash.ViashNamespace object NextflowHelper { + import io.viash.config.encodeConfig + private def readSource(s: String) = { val path = s"io/viash/runners/nextflow/$s" Source.fromResource(path).getLines().mkString("\n") @@ -44,7 +46,7 @@ object NextflowHelper { lazy val dataflowHelper: String = readSource("DataflowHelper.nf") def generateConfigStr(config: Config): String = { - val configJson = Json.Null//config.asJson.dropEmptyRecursively + val configJson = config.asJson.dropEmptyRecursively val configJsonStr = configJson .toFormattedString("json") .replace("\\\\", "\\\\\\\\") @@ -89,10 +91,10 @@ object NextflowHelper { def generateDefaultWorkflowArgs(config: Config, directives: NextflowDirectives, auto: NextflowAuto, debug: Boolean): String = { // override container val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) - val dirJson = Json.Null//directives.asJson.dropEmptyRecursively + val dirJson = directives.asJson.dropEmptyRecursively val dirJson2 = if (dirJson.isNull) Json.obj() else dirJson - val autoJson = Json.Null//auto.asJson.dropEmptyRecursively + val autoJson = auto.asJson.dropEmptyRecursively s"""[ | // key to be used to trace the process and determine output names From e30ee82c51072289014c4495545d35e38e81403b Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 29 Aug 2024 17:55:34 +0200 Subject: [PATCH 06/46] Fix another handful of issues, start to fix some tests --- .../io/viash/config/arguments/package.scala | 80 +++++++++---------- .../circe/DeriveConfiguredDecoder.scala | 25 ++++++ .../circe/DeriveConfiguredEncoder.scala | 1 - .../io/viash/TestingAllComponentsSuite.scala | 1 + ...inBuildAuxiliaryNativeParameterCheck.scala | 1 + ...BuildAuxiliaryNativeUnknownParameter.scala | 1 + .../auxiliary/MainRunVersionSwitch.scala | 2 +- .../viash/config/arguments/MergingTest.scala | 1 + .../config/arguments/StringArgumentTest.scala | 1 + .../io/viash/e2e/export/MainExportSuite.scala | 2 +- .../e2e/export/MainExportValidation.scala | 2 +- .../e2e/ns_build/MainNSBuildNativeSuite.scala | 1 + .../e2e/ns_list/MainNSListNativeSuite.scala | 1 + .../io/viash/e2e/run/MainRunDockerSuite.scala | 2 +- .../viash/e2e/test/MainTestNativeSuite.scala | 2 +- .../viash/escaping/EscapingNativeTest.scala | 1 + .../io/viash/helpers/circe/Convert.scala | 10 ++- .../viash/helpers/circe/ParseEitherTest.scala | 4 +- .../helpers/circe/ParseOneOrMoreTest.scala | 3 +- .../helpers/circe/ParseStringLikeTest.scala | 3 +- .../viash/helpers/circe/ValidationTest.scala | 3 +- 21 files changed, 92 insertions(+), 55 deletions(-) create mode 100644 src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala diff --git a/src/main/scala/io/viash/config/arguments/package.scala b/src/main/scala/io/viash/config/arguments/package.scala index 52f30329e..a508497cc 100644 --- a/src/main/scala/io/viash/config/arguments/package.scala +++ b/src/main/scala/io/viash/config/arguments/package.scala @@ -18,12 +18,10 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen -// import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ -// import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ -// import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ -import io.circe.generic.semiauto.{deriveDecoder as deriveConfiguredDecoderFullChecks, deriveEncoder as deriveConfiguredEncoderStrict} +import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ +import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ +import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ import io.viash.exceptions.ConfigParserSubTypeException package object arguments { @@ -64,54 +62,54 @@ package object arguments { } // encoders and decoders for Argument - // implicit val encodeStringArgument: Encoder.AsObject[StringArgument] = deriveConfiguredEncoderStrict[StringArgument] - // implicit val encodeIntegerArgument: Encoder.AsObject[IntegerArgument] = deriveConfiguredEncoderStrict[IntegerArgument] - // implicit val encodeLongArgument: Encoder.AsObject[LongArgument] = deriveConfiguredEncoderStrict[LongArgument] - // implicit val encodeDoubleArgument: Encoder.AsObject[DoubleArgument] = deriveConfiguredEncoderStrict[DoubleArgument] - // implicit val encodeBooleanArgumentR: Encoder.AsObject[BooleanArgument] = deriveConfiguredEncoderStrict[BooleanArgument] - // implicit val encodeBooleanArgumentT: Encoder.AsObject[BooleanTrueArgument] = deriveConfiguredEncoderStrict[BooleanTrueArgument] - // implicit val encodeBooleanArgumentF: Encoder.AsObject[BooleanFalseArgument] = deriveConfiguredEncoderStrict[BooleanFalseArgument] - // implicit val encodeFileArgument: Encoder.AsObject[FileArgument] = deriveConfiguredEncoderStrict[FileArgument] + implicit val encodeStringArgument: Encoder.AsObject[StringArgument] = deriveConfiguredEncoderStrict[StringArgument] + implicit val encodeIntegerArgument: Encoder.AsObject[IntegerArgument] = deriveConfiguredEncoderStrict[IntegerArgument] + implicit val encodeLongArgument: Encoder.AsObject[LongArgument] = deriveConfiguredEncoderStrict[LongArgument] + implicit val encodeDoubleArgument: Encoder.AsObject[DoubleArgument] = deriveConfiguredEncoderStrict[DoubleArgument] + implicit val encodeBooleanArgumentR: Encoder.AsObject[BooleanArgument] = deriveConfiguredEncoderStrict[BooleanArgument] + implicit val encodeBooleanArgumentT: Encoder.AsObject[BooleanTrueArgument] = deriveConfiguredEncoderStrict[BooleanTrueArgument] + implicit val encodeBooleanArgumentF: Encoder.AsObject[BooleanFalseArgument] = deriveConfiguredEncoderStrict[BooleanFalseArgument] + implicit val encodeFileArgument: Encoder.AsObject[FileArgument] = deriveConfiguredEncoderStrict[FileArgument] implicit def encodeArgument[A <: Argument[_]]: Encoder[A] = Encoder.instance { par => val typeJson = Json.obj("type" -> Json.fromString(par.`type`)) - // val objJson = par match { - // case s: StringArgument => encodeStringArgument(s) - // case s: IntegerArgument => encodeIntegerArgument(s) - // case s: LongArgument => encodeLongArgument(s) - // case s: DoubleArgument => encodeDoubleArgument(s) - // case s: BooleanArgument => encodeBooleanArgumentR(s) - // case s: BooleanTrueArgument => encodeBooleanArgumentT(s) - // case s: BooleanFalseArgument => encodeBooleanArgumentF(s) - // case s: FileArgument => encodeFileArgument(s) - // } - // objJson deepMerge typeJson + val objJson = par match { + case s: StringArgument => encodeStringArgument(s) + case s: IntegerArgument => encodeIntegerArgument(s) + case s: LongArgument => encodeLongArgument(s) + case s: DoubleArgument => encodeDoubleArgument(s) + case s: BooleanArgument => encodeBooleanArgumentR(s) + case s: BooleanTrueArgument => encodeBooleanArgumentT(s) + case s: BooleanFalseArgument => encodeBooleanArgumentF(s) + case s: FileArgument => encodeFileArgument(s) + } + objJson deepMerge typeJson typeJson } - // implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeIntegerArgument: Decoder[IntegerArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeLongArgument: Decoder[LongArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeDoubleArgument: Decoder[DoubleArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeBooleanArgumentR: Decoder[BooleanArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeBooleanArgumentT: Decoder[BooleanTrueArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeBooleanArgumentF: Decoder[BooleanFalseArgument] = deriveConfiguredDecoderFullChecks - // implicit val decodeFileArgument: Decoder[FileArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeIntegerArgument: Decoder[IntegerArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeLongArgument: Decoder[LongArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeDoubleArgument: Decoder[DoubleArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeBooleanArgumentR: Decoder[BooleanArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeBooleanArgumentT: Decoder[BooleanTrueArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeBooleanArgumentF: Decoder[BooleanFalseArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeFileArgument: Decoder[FileArgument] = deriveConfiguredDecoderFullChecks implicit def decodeDataArgument: Decoder[Argument[_]] = Decoder.instance { cursor => val decoder: Decoder[Argument[_]] = cursor.downField("type").as[String] match { - // case Right("string") => decodeStringArgument.widen - // case Right("integer") => decodeIntegerArgument.widen - // case Right("long") => decodeLongArgument.widen - // case Right("double") => decodeDoubleArgument.widen - // case Right("boolean") => decodeBooleanArgumentR.widen - // case Right("boolean_true") => decodeBooleanArgumentT.widen - // case Right("boolean_false") => decodeBooleanArgumentF.widen - // case Right("file") => decodeFileArgument.widen - // case Right(typ) => invalidSubTypeDecoder[StringArgument](typ, List("string", "integer", "long", "double", "boolean", "boolean_true", "boolean_false", "file")).widen + case Right("string") => decodeStringArgument.widen + case Right("integer") => decodeIntegerArgument.widen + case Right("long") => decodeLongArgument.widen + case Right("double") => decodeDoubleArgument.widen + case Right("boolean") => decodeBooleanArgumentR.widen + case Right("boolean_true") => decodeBooleanArgumentT.widen + case Right("boolean_false") => decodeBooleanArgumentF.widen + case Right("file") => decodeFileArgument.widen + case Right(typ) => invalidSubTypeDecoder[StringArgument](typ, List("string", "integer", "long", "double", "boolean", "boolean_true", "boolean_false", "file")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala new file mode 100644 index 000000000..b8c217ccf --- /dev/null +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers.circe + +import io.circe.derivation.{Configuration, ConfiguredDecoder} +import scala.deriving.Mirror + +object DeriveConfiguredDecoder { + inline def deriveConfiguredDecoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] +} \ No newline at end of file diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala index 34d2ab77a..b8859e38b 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala @@ -17,7 +17,6 @@ package io.viash.helpers.circe -import io.circe.Encoder import io.circe.derivation.{Configuration, ConfiguredEncoder} import scala.deriving.Mirror diff --git a/src/test/scala/io/viash/TestingAllComponentsSuite.scala b/src/test/scala/io/viash/TestingAllComponentsSuite.scala index 6a64f8762..44a9053c5 100644 --- a/src/test/scala/io/viash/TestingAllComponentsSuite.scala +++ b/src/test/scala/io/viash/TestingAllComponentsSuite.scala @@ -5,6 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite import io.viash.helpers.Logger import org.scalatest.ParallelTestExecution import io.viash.lenses.ConfigLenses +import io.viash.config.{decodeConfig, encodeConfig} class TestingAllComponentsSuite extends AnyFunSuite with ParallelTestExecution { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala index 97a5eb268..20e7e348c 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala @@ -11,6 +11,7 @@ import io.viash.helpers.{IO, Exec, Logger} import io.viash.TestHelper import java.nio.file.Path import scala.annotation.meta.param +import io.viash.helpers.data_structures._ class MainBuildAuxiliaryNativeParameterCheck extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala index 44e5e4ce4..1504aef14 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala @@ -9,6 +9,7 @@ import io.viash.config.Config import scala.io.Source import io.viash.helpers.{IO, Exec, Logger} import io.viash.TestHelper +import io.viash.helpers.data_structures._ class MainBuildAuxiliaryNativeUnknownParameter extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala index 3064e6955..3b72e26d5 100644 --- a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala +++ b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala @@ -6,7 +6,7 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -import scala.reflect.io.Directory +// import scala.reflect.io.Directory import java.io.ByteArrayOutputStream import io.viash.exceptions.ExitException diff --git a/src/test/scala/io/viash/config/arguments/MergingTest.scala b/src/test/scala/io/viash/config/arguments/MergingTest.scala index dff7331da..ca98ddaca 100644 --- a/src/test/scala/io/viash/config/arguments/MergingTest.scala +++ b/src/test/scala/io/viash/config/arguments/MergingTest.scala @@ -5,6 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite import io.viash.helpers.Logger import io.viash.helpers.circe.Convert import io.viash.config.Config +import io.viash.config.decodeConfig class MergingTest extends AnyFunSuite { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala b/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala index a87be837d..934c651f5 100644 --- a/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala +++ b/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala @@ -10,6 +10,7 @@ import io.circe.yaml.{parser => YamlParser} import io.viash.helpers.circe._ import io.viash.helpers.data_structures._ import io.viash.helpers.Logger +import io.viash.config.arguments.decodeStringArgument class StringArgumentTest extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala index 560ba0cf8..d697a3990 100644 --- a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala +++ b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala @@ -1,4 +1,4 @@ -package io.viash.e2e.export +package io.viash.e2e.`export` import io.viash._ import io.viash.helpers.Logger diff --git a/src/test/scala/io/viash/e2e/export/MainExportValidation.scala b/src/test/scala/io/viash/e2e/export/MainExportValidation.scala index f2d70169b..bc1a4552b 100644 --- a/src/test/scala/io/viash/e2e/export/MainExportValidation.scala +++ b/src/test/scala/io/viash/e2e/export/MainExportValidation.scala @@ -1,4 +1,4 @@ -package io.viash.e2e.export +package io.viash.e2e.`export` import io.viash._ import io.viash.helpers.{Logger, IO} diff --git a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala index a3c4a857d..4779023f2 100644 --- a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala @@ -11,6 +11,7 @@ import java.io.File import java.nio.file.Paths import scala.io.Source import java.io.ByteArrayOutputStream +import io.viash.helpers.data_structures._ class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{ Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala index 63fe346c0..6470400d6 100644 --- a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala @@ -11,6 +11,7 @@ import org.scalatest.funsuite.AnyFunSuite import java.io.File import java.nio.file.Paths import scala.io.Source +import io.viash.config.decodeConfig class MainNSListNativeSuite extends AnyFunSuite{ Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala index 43c6b74fb..9a536b523 100644 --- a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala +++ b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala @@ -8,7 +8,7 @@ import io.viash.helpers.IO import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite -import scala.reflect.io.Directory +// import scala.reflect.io.Directory import sys.process._ class MainRunDockerSuite extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala index 48422d054..5be43bdc7 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala @@ -8,7 +8,7 @@ import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite -import scala.reflect.io.Directory +// import scala.reflect.io.Directory import sys.process._ import io.viash.exceptions.ConfigParserException import io.viash.exceptions.MissingResourceFileException diff --git a/src/test/scala/io/viash/escaping/EscapingNativeTest.scala b/src/test/scala/io/viash/escaping/EscapingNativeTest.scala index 9f5544253..8c2ba48f3 100644 --- a/src/test/scala/io/viash/escaping/EscapingNativeTest.scala +++ b/src/test/scala/io/viash/escaping/EscapingNativeTest.scala @@ -9,6 +9,7 @@ import java.io.{IOException, UncheckedIOException} import java.nio.file.{Files, Path, Paths} import scala.io.Source import io.viash.helpers.{IO, Exec, Logger} +import io.viash.helpers.data_structures._ class EscapingNativeTest extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/helpers/circe/Convert.scala b/src/test/scala/io/viash/helpers/circe/Convert.scala index 8a00dc324..f0ae9cee4 100644 --- a/src/test/scala/io/viash/helpers/circe/Convert.scala +++ b/src/test/scala/io/viash/helpers/circe/Convert.scala @@ -5,12 +5,14 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.viash.exceptions.{ConfigYamlException, ConfigParserException} -import shapeless.Lazy -import scala.reflect.runtime.universe._ +// import shapeless.Lazy +// import scala.reflect.runtime.universe._ import io.circe.Decoder -import io.circe.generic.extras.decoding.ConfiguredDecoder -import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +// import io.circe.generic.extras.decoding.ConfiguredDecoder +// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +import io.viash.helpers.circe.DeriveConfiguredDecoder._ +import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.Logger class ConvertTest extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala index 60548d687..b881900ab 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala @@ -4,7 +4,9 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +//import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.circe.DeriveConfiguredEncoder._ +import io.viash.helpers.circe.DeriveConfiguredDecoder._ import io.viash.helpers.Logger class ParseEitherTest extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala index 763a01c3c..4d0ba44aa 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala @@ -5,7 +5,8 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser import io.viash.helpers.circe._ -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.circe.DeriveConfiguredDecoder._ +import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.data_structures._ import io.viash.helpers.Logger diff --git a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala index 063b6c7cd..a2f570d25 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala @@ -4,7 +4,8 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.circe.DeriveConfiguredDecoder._ +import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.Logger class ParseStringLikeTest extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala index 182ef9a59..b2de9da4e 100644 --- a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala @@ -5,7 +5,8 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser import io.viash.helpers.circe._ -import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.circe.DeriveConfiguredDecoder._ +import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.data_structures._ import io.viash.helpers.Logger import java.io.ByteArrayOutputStream From c9d10b805e6bd49cf7f9754e4c6ef3197608921e Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 29 Aug 2024 20:03:32 +0200 Subject: [PATCH 07/46] Fix another bunch of issues with testbenches --- .../viash/config/dependencies/package.scala | 1 - ...onfiguredDecoderWithDeprecationCheck.scala | 7 +++---- ...ConfiguredDecoderWithValidationCheck.scala | 8 +++----- .../auxiliary/MainRunVersionSwitch.scala | 1 - .../MainTestAuxiliaryDockerResourceCopy.scala | 19 +++++++++---------- .../io/viash/e2e/build/DockerSuite.scala | 1 + .../io/viash/e2e/build/NativeSuite.scala | 1 + .../io/viash/e2e/run/MainRunDockerSuite.scala | 1 - .../viash/e2e/test/MainTestDockerSuite.scala | 17 ++++++++--------- .../viash/e2e/test/MainTestNativeSuite.scala | 15 +++++++-------- 10 files changed, 32 insertions(+), 39 deletions(-) diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index 7abe7672e..efe36b8fb 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -18,7 +18,6 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import cats.syntax.functor._ import dependencies.GithubRepository diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 1269c0841..477c31de5 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -17,12 +17,10 @@ package io.viash.helpers.circe -// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder import io.circe.{ Decoder, CursorOp } -// import io.circe.generic.extras.decoding.ConfiguredDecoder -// import scala.reflect.runtime.universe._ -// import shapeless.Lazy +import io.circe.derivation.{Configuration, ConfiguredDecoder} +import scala.deriving.Mirror import io.viash.schemas.ParameterSchema import io.circe.ACursor @@ -100,4 +98,5 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here // def deriveConfiguredDecoderWithDeprecationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] // .prepare( checkDeprecation[A] ) + inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 7bb3ee61b..af423f5ea 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -17,13 +17,10 @@ package io.viash.helpers.circe -// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder import io.circe.{ Decoder, CursorOp } -// import io.circe.generic.extras.decoding.ConfiguredDecoder - -// import scala.reflect.runtime.universe._ -// import shapeless.Lazy +import io.circe.derivation.{Configuration, ConfiguredDecoder} +import scala.deriving.Mirror import io.viash.schemas.ParameterSchema import io.circe.ACursor import io.viash.exceptions.ConfigParserSubTypeException @@ -80,6 +77,7 @@ object DeriveConfiguredDecoderWithValidationCheck { // }, // s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." // ) + inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String]): Decoder[A] = Decoder.instance { cursor => throw new ConfigParserSubTypeException(tpe, validTypes, cursor.value.toString()) } diff --git a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala index 3b72e26d5..3cea33235 100644 --- a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala +++ b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala @@ -6,7 +6,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -// import scala.reflect.io.Directory import java.io.ByteArrayOutputStream import io.viash.exceptions.ExitException diff --git a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala index f7bf653a1..e04f52d94 100644 --- a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala +++ b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala @@ -6,7 +6,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -import scala.reflect.io.Directory import io.viash.ConfigDeriver class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfterAll { @@ -78,7 +77,7 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte assert(md5sum.r.findFirstMatchIn(hash).isDefined, s"Calculated md5sum doesn't match the given md5sum for $name") } - Directory(tmpFolderResourceDestinationFolder).deleteRecursively() + IO.deleteRecursively(tmpFolderResourceDestinationFolder.toPath) checkTempDirAndRemove(testOutput.stdout, true, "viash_test_auxiliary_resources") } @@ -109,30 +108,30 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte * @param expectDirectoryExists expect the directory to be present or not * @return */ - def checkTempDirAndRemove(testText: String, expectDirectoryExists: Boolean, folderName: String = "viash_test_testbash"): Unit = { + def checkTempDirAndRemove(testText: String, expectDirectoryExists: Boolean, testDirName: String = "viash_test_testbash"): Unit = { // Get temporary directory val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r - val tempPath = testText.replaceAll("\n", "") match { + val tempPathStr = testText.replaceAll("\n", "") match { case FolderRegex(path) => path case _ => "" } - assert(tempPath.contains(s"${IO.tempDir}/$folderName")) + assert(tempPathStr.contains(s"${IO.tempDir}/$testDirName")) - val tempFolder = new Directory(Paths.get(tempPath).toFile) + val tempPath = Paths.get(tempPathStr) if (expectDirectoryExists) { // Check temporary directory is still present - assert(tempFolder.exists) - assert(tempFolder.isDirectory) + assert(Files.exists(tempPath)) + assert(Files.isDirectory(tempPath)) // Remove the temporary directory - tempFolder.deleteRecursively() + IO.deleteRecursively(tempPath) } // folder should always have been removed at this stage - assert(!tempFolder.exists) + assert(!Files.exists(tempPath)) } override def afterAll(): Unit = { diff --git a/src/test/scala/io/viash/e2e/build/DockerSuite.scala b/src/test/scala/io/viash/e2e/build/DockerSuite.scala index 9b21ccbbd..90b60da8a 100644 --- a/src/test/scala/io/viash/e2e/build/DockerSuite.scala +++ b/src/test/scala/io/viash/e2e/build/DockerSuite.scala @@ -10,6 +10,7 @@ import io.viash.helpers.{IO, Exec, Logger} import io.viash.config.Config import scala.io.Source +import io.viash.helpers.data_structures._ class DockerSuite extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/e2e/build/NativeSuite.scala b/src/test/scala/io/viash/e2e/build/NativeSuite.scala index 49ada6588..b3a561379 100644 --- a/src/test/scala/io/viash/e2e/build/NativeSuite.scala +++ b/src/test/scala/io/viash/e2e/build/NativeSuite.scala @@ -12,6 +12,7 @@ import scala.io.Source import io.viash.helpers.{IO, Exec, Logger} import io.viash.exceptions.ConfigParserException import java.nio.file.Files +import io.viash.helpers.data_structures._ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { Logger.UseColorOverride.value = Some(false) diff --git a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala index 9a536b523..a4650cf28 100644 --- a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala +++ b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala @@ -8,7 +8,6 @@ import io.viash.helpers.IO import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite -// import scala.reflect.io.Directory import sys.process._ class MainRunDockerSuite extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala index 44ffd466b..2c33a0771 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala @@ -8,10 +8,9 @@ import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite -import scala.reflect.io.Directory import sys.process._ import org.scalatest.ParallelTestExecution -import java.nio.file.Path +import java.nio.file.{Path, Files} class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with ParallelTestExecution{ Logger.UseColorOverride.value = Some(false) @@ -336,26 +335,26 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall // Get temporary directory val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r - val tempPath = testText.replaceAll("\n", "") match { + val tempPathStr = testText.replaceAll("\n", "") match { case FolderRegex(path) => path case _ => "" } - assert(tempPath.contains(s"${IO.tempDir}/viash_test_testbash")) + assert(tempPathStr.contains(s"${IO.tempDir}/viash_test_testbash")) - val tempFolder = new Directory(Paths.get(tempPath).toFile) + val tempPath = Paths.get(tempPathStr) if (expectDirectoryExists) { // Check temporary directory is still present - assert(tempFolder.exists) - assert(tempFolder.isDirectory) + assert(Files.exists(tempPath)) + assert(Files.isDirectory(tempPath)) // Remove the temporary directory - tempFolder.deleteRecursively() + IO.deleteRecursively(tempPath) } // folder should always have been removed at this stage - assert(!tempFolder.exists) + assert(!Files.exists(tempPath)) } override def afterAll(): Unit = { diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala index 5be43bdc7..771d5f4a5 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala @@ -8,7 +8,6 @@ import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite -// import scala.reflect.io.Directory import sys.process._ import io.viash.exceptions.ConfigParserException import io.viash.exceptions.MissingResourceFileException @@ -563,26 +562,26 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { // Get temporary directory val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r - val tempPath = testText.replaceAll("\n", "") match { + val tempPathStr = testText.replaceAll("\n", "") match { case FolderRegex(path) => path case _ => "" } - assert(tempPath.contains(s"${IO.tempDir}/$testDirName")) + assert(tempPathStr.contains(s"${IO.tempDir}/$testDirName")) - val tempFolder = new Directory(Paths.get(tempPath).toFile) + val tempPath = Paths.get(tempPathStr) if (expectDirectoryExists) { // Check temporary directory is still present - assert(tempFolder.exists) - assert(tempFolder.isDirectory) + assert(Files.exists(tempPath)) + assert(Files.isDirectory(tempPath)) // Remove the temporary directory - tempFolder.deleteRecursively() + IO.deleteRecursively(tempPath) } // folder should always have been removed at this stage - assert(!tempFolder.exists) + assert(!Files.exists(tempPath)) } override def afterAll(): Unit = { From b00815411198bac42a27dc1704fbcab0b0ed83e1 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Sep 2024 11:11:34 +0200 Subject: [PATCH 08/46] Re-enable code & provide temporary methods as needed. Further cleanup --- src/main/scala/io/viash/cli/package.scala | 2 -- src/main/scala/io/viash/config/Config.scala | 2 +- .../io/viash/config/arguments/package.scala | 4 +-- .../viash/config/dependencies/package.scala | 2 -- src/main/scala/io/viash/config/package.scala | 23 +++++------------ .../io/viash/config/resources/package.scala | 6 +---- src/main/scala/io/viash/engines/package.scala | 3 --- .../viash/engines/requirements/package.scala | 2 -- .../io/viash/functionality/package.scala | 4 --- .../circe/DeriveConfiguredDecoder.scala | 25 ------------------- ...onfiguredDecoderWithDeprecationCheck.scala | 4 +++ .../circe/DeriveConfiguredEncoder.scala | 25 ------------------- .../io/viash/helpers/circe/package.scala | 8 +++++- .../io/viash/packageConfig/package.scala | 3 +-- .../scala/io/viash/platforms/package.scala | 4 --- .../io/viash/runners/nextflow/package.scala | 2 -- src/main/scala/io/viash/runners/package.scala | 3 --- .../io/viash/schemas/CollectedSchemas.scala | 3 --- .../io/viash/helpers/circe/Convert.scala | 7 ------ .../viash/helpers/circe/ParseEitherTest.scala | 3 --- .../helpers/circe/ParseOneOrMoreTest.scala | 2 -- .../helpers/circe/ParseStringLikeTest.scala | 2 -- .../viash/helpers/circe/ValidationTest.scala | 2 -- 23 files changed, 21 insertions(+), 120 deletions(-) delete mode 100644 src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala delete mode 100644 src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala diff --git a/src/main/scala/io/viash/cli/package.scala b/src/main/scala/io/viash/cli/package.scala index 1b2e4931b..586ddc643 100644 --- a/src/main/scala/io/viash/cli/package.scala +++ b/src/main/scala/io/viash/cli/package.scala @@ -18,8 +18,6 @@ package io.viash import io.circe.Encoder -// import io.circe.generic.extras.semiauto.deriveConfiguredEncoder -import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoder import org.rogach.scallop.CliOption package object cli { diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index dd96e5d6a..28d025b15 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -687,7 +687,7 @@ object Config extends Logging { // apply config mods val modifiedJs = confMods(js, preparse = false) // turn json back into a config - Config("")//Convert.jsonToClass[Config](modifiedJs, uri.toString()) + Convert.jsonToClass[Config](modifiedJs, uri.toString()) } else { conf0 } diff --git a/src/main/scala/io/viash/config/arguments/package.scala b/src/main/scala/io/viash/config/arguments/package.scala index a508497cc..083e341fc 100644 --- a/src/main/scala/io/viash/config/arguments/package.scala +++ b/src/main/scala/io/viash/config/arguments/package.scala @@ -19,9 +19,7 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} import cats.syntax.functor._ // for .widen -import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ -import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ -import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ +import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder import io.viash.exceptions.ConfigParserSubTypeException package object arguments { diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index efe36b8fb..9b27eb584 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -18,14 +18,12 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import cats.syntax.functor._ import dependencies.GithubRepository package object dependencies { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // encoders and decoders for Argument implicit val encodeEitherStringRepository: Encoder[Either[String, Repository]] = deriveConfiguredEncoderStrict diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index 68cfdfadd..0453765d1 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -18,15 +18,6 @@ package io.viash import io.circe.{Decoder, Encoder, Json, HCursor, JsonObject} -// import io.circe.generic.extras.Configuration -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -// import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoder -// import io.circe.generic.semiauto.deriveEncoder as deriveConfiguredEncoderStrict -// import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoderFullChecks -// import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoderWithValidationCheck -// import io.viash.platforms.decodePlatform -// import io.viash.functionality.decodeFunctionality -import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.exceptions.ConfigParserValidationException import config.ArgumentGroup @@ -39,10 +30,8 @@ import config.arguments._ package object config { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation + import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck.deriveConfiguredDecoderWithValidationCheck import io.viash.config.resources.{decodeResource, encodeResource} import io.viash.config.dependencies.{decodeDependency, encodeDependency} @@ -50,11 +39,13 @@ package object config { import io.viash.runners.{decodeRunner, encodeRunner} import io.viash.engines.{decodeEngine, encodeEngine} import io.viash.packageConfig.{decodePackageConfig, encodePackageConfig} + import io.viash.platforms.decodePlatform + import io.viash.functionality.decodeFunctionality // encoders and decoders for Config implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] - // implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config] - /*.prepare{ + implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config] + .prepare{ checkDeprecation[Config](_) // map platforms to runners and engines .withFocus{ @@ -279,8 +270,6 @@ package object config { }, "Could not convert json to Config." ) - }*/ - implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderFullChecks implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/config/resources/package.scala b/src/main/scala/io/viash/config/resources/package.scala index 3257a7657..9a84004ee 100644 --- a/src/main/scala/io/viash/config/resources/package.scala +++ b/src/main/scala/io/viash/config/resources/package.scala @@ -18,17 +18,13 @@ package io.viash.config import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -import cats.syntax.functor._ +import cats.syntax.functor._ // for .widen -import java.net.URI // for .widen import io.circe.ACursor package object resources { - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ // implicit val encodeURI: Encoder[URI] = Encoder.instance { // uri => Json.fromString(uri.toString) diff --git a/src/main/scala/io/viash/engines/package.scala b/src/main/scala/io/viash/engines/package.scala index f61995ed4..e6ef6f1c0 100644 --- a/src/main/scala/io/viash/engines/package.scala +++ b/src/main/scala/io/viash/engines/package.scala @@ -18,14 +18,11 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen package object engines { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.engines.requirements.{decodeRequirements, encodeRequirements} diff --git a/src/main/scala/io/viash/engines/requirements/package.scala b/src/main/scala/io/viash/engines/requirements/package.scala index 554070f8a..d3acf176e 100644 --- a/src/main/scala/io/viash/engines/requirements/package.scala +++ b/src/main/scala/io/viash/engines/requirements/package.scala @@ -23,8 +23,6 @@ import cats.syntax.functor._ // for .widen package object requirements { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredEncoder._ implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/functionality/package.scala b/src/main/scala/io/viash/functionality/package.scala index dd1b91338..68dd893ee 100644 --- a/src/main/scala/io/viash/functionality/package.scala +++ b/src/main/scala/io/viash/functionality/package.scala @@ -36,10 +36,6 @@ import config.arguments._ package object functionality extends Logging { // import implicits import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ import io.viash.config.{decodeAuthor, decodeArgumentGroup, decodeStatus, decodeComputationalRequirements, decodeReferences, decodeLinks} import io.viash.config.resources.decodeResource diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala deleted file mode 100644 index b8c217ccf..000000000 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoder.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Data Intuitive - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package io.viash.helpers.circe - -import io.circe.derivation.{Configuration, ConfiguredDecoder} -import scala.deriving.Mirror - -object DeriveConfiguredDecoder { - inline def deriveConfiguredDecoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] -} \ No newline at end of file diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 477c31de5..78d86090e 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -95,6 +95,10 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { // cursor // return unchanged json info // } + def checkDeprecation[A](cursor: ACursor): ACursor = { + cursor + } + // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here // def deriveConfiguredDecoderWithDeprecationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] // .prepare( checkDeprecation[A] ) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala deleted file mode 100644 index b8859e38b..000000000 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoder.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2020 Data Intuitive - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package io.viash.helpers.circe - -import io.circe.derivation.{Configuration, ConfiguredEncoder} -import scala.deriving.Mirror - -object DeriveConfiguredEncoder { - inline def deriveConfiguredEncoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A] -} \ No newline at end of file diff --git a/src/main/scala/io/viash/helpers/circe/package.scala b/src/main/scala/io/viash/helpers/circe/package.scala index b44730a21..a11139f0e 100644 --- a/src/main/scala/io/viash/helpers/circe/package.scala +++ b/src/main/scala/io/viash/helpers/circe/package.scala @@ -18,15 +18,21 @@ package io.viash.helpers import io.circe._ -import io.circe.derivation.Configuration +import io.circe.derivation.{Configuration, ConfiguredDecoder, ConfiguredEncoder} import java.net.URI import data_structures.OneOrMore import java.nio.file.Paths +import scala.deriving.Mirror + package object circe { implicit val customConfig: Configuration = Configuration.default.withDefaults.withStrictDecoding + inline def deriveConfiguredDecoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] + inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = DeriveConfiguredDecoderFullChecks.deriveConfiguredDecoderFullChecks + inline def deriveConfiguredEncoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A] + inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = DeriveConfiguredEncoderStrict.deriveConfiguredEncoderStrict // // encoder and decoder for Either implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { diff --git a/src/main/scala/io/viash/packageConfig/package.scala b/src/main/scala/io/viash/packageConfig/package.scala index d33b3e5b6..f60d2505f 100644 --- a/src/main/scala/io/viash/packageConfig/package.scala +++ b/src/main/scala/io/viash/packageConfig/package.scala @@ -21,8 +21,7 @@ import io.circe.{Decoder, Encoder} package object packageConfig { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ + import io.viash.config.{decodeAuthor, encodeAuthor} import io.viash.config.{decodeLinks, encodeLinks} import io.viash.config.{decodeReferences, encodeReferences} diff --git a/src/main/scala/io/viash/platforms/package.scala b/src/main/scala/io/viash/platforms/package.scala index 23ae7231b..38053e900 100644 --- a/src/main/scala/io/viash/platforms/package.scala +++ b/src/main/scala/io/viash/platforms/package.scala @@ -18,14 +18,10 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -import io.circe.generic.semiauto.deriveDecoder as deriveConfiguredDecoder import cats.syntax.functor._ // for .widen package object platforms { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ - import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.runners.nextflow.{decodeNextflowDirectives, encodeNextflowDirectives} import io.viash.engines.docker.{decodeResolveVolume, encodeResolveVolume} diff --git a/src/main/scala/io/viash/runners/nextflow/package.scala b/src/main/scala/io/viash/runners/nextflow/package.scala index 320ab11a8..de6c7e70f 100644 --- a/src/main/scala/io/viash/runners/nextflow/package.scala +++ b/src/main/scala/io/viash/runners/nextflow/package.scala @@ -22,8 +22,6 @@ import io.circe.derivation.ConfiguredDecoder package object nextflow { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredEncoder._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/runners/package.scala b/src/main/scala/io/viash/runners/package.scala index a9925895e..591cea5de 100644 --- a/src/main/scala/io/viash/runners/package.scala +++ b/src/main/scala/io/viash/runners/package.scala @@ -18,14 +18,11 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import cats.syntax.functor._ // for .widen -import io.viash.helpers.circe.DeriveConfiguredEncoder.deriveConfiguredEncoder package object runners { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import io.viash.runners.executable.{decodeSetupStrategy, encodeSetupStrategy} import io.viash.runners.nextflow.{decodeNextflowDirectives, encodeNextflowDirectives} diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 8b6fa89fd..93376ca1a 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -20,8 +20,6 @@ package io.viash.schemas // import scala.reflect.runtime.universe._ import io.circe.{Encoder, Printer => JsonPrinter} import io.circe.syntax.EncoderOps -// import io.circe.generic.extras.semiauto.deriveConfiguredEncoder -import io.circe.generic.semiauto.{deriveEncoder as deriveConfiguredEncoder} import io.viash.functionality._ import io.viash.runners._ @@ -77,7 +75,6 @@ object CollectedSchemas { private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._ private implicit val encodeConfigSchema: Encoder.AsObject[CollectedSchemas] = deriveConfiguredEncoder private implicit val encodeParameterSchema: Encoder.AsObject[ParameterSchema] = deriveConfiguredEncoderStrict diff --git a/src/test/scala/io/viash/helpers/circe/Convert.scala b/src/test/scala/io/viash/helpers/circe/Convert.scala index f0ae9cee4..46621d750 100644 --- a/src/test/scala/io/viash/helpers/circe/Convert.scala +++ b/src/test/scala/io/viash/helpers/circe/Convert.scala @@ -5,14 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.viash.exceptions.{ConfigYamlException, ConfigParserException} -// import shapeless.Lazy -// import scala.reflect.runtime.universe._ - import io.circe.Decoder -// import io.circe.generic.extras.decoding.ConfiguredDecoder -// import io.circe.generic.extras.semiauto.deriveConfiguredDecoder -import io.viash.helpers.circe.DeriveConfiguredDecoder._ -import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.Logger class ConvertTest extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala index b881900ab..d45de5844 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala @@ -4,9 +4,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser -//import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} -import io.viash.helpers.circe.DeriveConfiguredEncoder._ -import io.viash.helpers.circe.DeriveConfiguredDecoder._ import io.viash.helpers.Logger class ParseEitherTest extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala index 4d0ba44aa..8c58a95cf 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala @@ -5,8 +5,6 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser import io.viash.helpers.circe._ -import io.viash.helpers.circe.DeriveConfiguredDecoder._ -import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.data_structures._ import io.viash.helpers.Logger diff --git a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala index a2f570d25..28791e04e 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala @@ -4,8 +4,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser -import io.viash.helpers.circe.DeriveConfiguredDecoder._ -import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.Logger class ParseStringLikeTest extends AnyFunSuite with BeforeAndAfterAll { diff --git a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala index b2de9da4e..126338d20 100644 --- a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala @@ -5,8 +5,6 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser import io.viash.helpers.circe._ -import io.viash.helpers.circe.DeriveConfiguredDecoder._ -import io.viash.helpers.circe.DeriveConfiguredEncoder._ import io.viash.helpers.data_structures._ import io.viash.helpers.Logger import java.io.ByteArrayOutputStream From 664a782b0b7b0f59619130266bd578776126c60a Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Sep 2024 11:42:40 +0200 Subject: [PATCH 09/46] minor cleanup --- .../scala/io/viash/config/dependencies/package.scala | 4 ++-- src/main/scala/io/viash/config/package.scala | 11 ++++------- .../scala/io/viash/functionality/Functionality.scala | 1 - src/main/scala/io/viash/functionality/package.scala | 1 - src/main/scala/io/viash/helpers/circe/package.scala | 2 +- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index 9b27eb584..7b880fb29 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -26,7 +26,7 @@ package object dependencies { import io.viash.helpers.circe._ // encoders and decoders for Argument - implicit val encodeEitherStringRepository: Encoder[Either[String, Repository]] = deriveConfiguredEncoderStrict + // implicit val encodeEitherStringRepository: Encoder[Either[String, Repository]] = deriveConfiguredEncoderStrict implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict @@ -65,7 +65,7 @@ package object dependencies { objJson deepMerge typeJson } - implicit val decodeEitherStringRepository: Decoder[Either[String, Repository]] = deriveConfiguredDecoderFullChecks + // implicit val decodeEitherStringRepository: Decoder[Either[String, Repository]] = deriveConfiguredDecoderFullChecks implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index 0453765d1..63690be5a 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -44,8 +44,7 @@ package object config { // encoders and decoders for Config implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config] - implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config] - .prepare{ + implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithValidationCheck[Config].prepare{ checkDeprecation[Config](_) // map platforms to runners and engines .withFocus{ @@ -288,11 +287,9 @@ package object config { // encoder and decoder for Status, make string lowercase before decoding implicit val encodeStatus: Encoder[Status] = deriveConfiguredEncoder - implicit val decodeStatus: Decoder[Status] = deriveConfiguredDecoderFullChecks - // implicit val encodeStatus: Encoder[Status] = Encoder.encodeEnumeration(Status) - // implicit val decodeStatus: Decoder[Status] = Decoder.decodeEnumeration(Status).prepare { - // _.withFocus(_.mapString(_.toLowerCase())) - // } + implicit val decodeStatus: Decoder[Status] = deriveConfiguredDecoderFullChecks[Status].prepare { + _.withFocus(_.mapString(_.toLowerCase())) + } implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala index d455be4b0..43c0ca993 100644 --- a/src/main/scala/io/viash/functionality/Functionality.scala +++ b/src/main/scala/io/viash/functionality/Functionality.scala @@ -19,7 +19,6 @@ package io.viash.functionality import io.circe.Json -// import io.circe.generic.extras._ import io.viash.config.arguments._ import io.viash.config.resources._ import io.viash.config.Status._ diff --git a/src/main/scala/io/viash/functionality/package.scala b/src/main/scala/io/viash/functionality/package.scala index 68dd893ee..61eaf22aa 100644 --- a/src/main/scala/io/viash/functionality/package.scala +++ b/src/main/scala/io/viash/functionality/package.scala @@ -17,7 +17,6 @@ package io.viash -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.circe.{Decoder, Encoder, Json} import io.circe.ACursor diff --git a/src/main/scala/io/viash/helpers/circe/package.scala b/src/main/scala/io/viash/helpers/circe/package.scala index a11139f0e..20163aa5c 100644 --- a/src/main/scala/io/viash/helpers/circe/package.scala +++ b/src/main/scala/io/viash/helpers/circe/package.scala @@ -34,7 +34,7 @@ package object circe { inline def deriveConfiguredEncoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A] inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = DeriveConfiguredEncoderStrict.deriveConfiguredEncoderStrict - // // encoder and decoder for Either + // encoder and decoder for Either implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = { _.fold(ea(_), eb(_)) } From dfdf294d8c7a3a8e6795d1f3b059af5566864a02 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Sep 2024 12:05:41 +0200 Subject: [PATCH 10/46] fix temporary code still being present fix platforms being able to encode fix executablerunner not using the correct encoder other small cleanup --- .../io/viash/config/arguments/package.scala | 1 - .../scala/io/viash/platforms/package.scala | 38 +++++++++---------- .../io/viash/runners/executable/package.scala | 1 - .../io/viash/runners/nextflow/package.scala | 1 - src/main/scala/io/viash/runners/package.scala | 5 +-- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/main/scala/io/viash/config/arguments/package.scala b/src/main/scala/io/viash/config/arguments/package.scala index 083e341fc..77a2374ff 100644 --- a/src/main/scala/io/viash/config/arguments/package.scala +++ b/src/main/scala/io/viash/config/arguments/package.scala @@ -83,7 +83,6 @@ package object arguments { case s: FileArgument => encodeFileArgument(s) } objJson deepMerge typeJson - typeJson } implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/platforms/package.scala b/src/main/scala/io/viash/platforms/package.scala index 38053e900..c83ceeb70 100644 --- a/src/main/scala/io/viash/platforms/package.scala +++ b/src/main/scala/io/viash/platforms/package.scala @@ -23,32 +23,32 @@ import cats.syntax.functor._ // for .widen package object platforms { import io.viash.helpers.circe._ - import io.viash.runners.nextflow.{decodeNextflowDirectives, encodeNextflowDirectives} - import io.viash.engines.docker.{decodeResolveVolume, encodeResolveVolume} - import io.viash.runners.executable.{decodeSetupStrategy, encodeSetupStrategy} - import io.viash.engines.requirements.{decodeRequirements, encodeRequirements} - import io.viash.runners.nextflow.{decodeNextflowAuto, encodeNextflowAuto} - import io.viash.runners.nextflow.{decodeNextflowConfig, encodeNextflowConfig} + import io.viash.runners.nextflow.decodeNextflowDirectives + import io.viash.engines.docker.decodeResolveVolume + import io.viash.runners.executable.decodeSetupStrategy + import io.viash.engines.requirements.decodeRequirements + import io.viash.runners.nextflow.decodeNextflowAuto + import io.viash.runners.nextflow.decodeNextflowConfig - implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder + // implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks - implicit val encodeNextflowPlatform: Encoder.AsObject[NextflowPlatform] = deriveConfiguredEncoder + // implicit val encodeNextflowPlatform: Encoder.AsObject[NextflowPlatform] = deriveConfiguredEncoder implicit val decodeNextflowPlatform: Decoder[NextflowPlatform] = deriveConfiguredDecoderFullChecks - implicit val encodeNativePlatform: Encoder.AsObject[NativePlatform] = deriveConfiguredEncoder + // implicit val encodeNativePlatform: Encoder.AsObject[NativePlatform] = deriveConfiguredEncoder implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderFullChecks - implicit def encodePlatform[A <: Platform]: Encoder[A] = Encoder.instance { - platform => - val typeJson = Json.obj("type" -> Json.fromString(platform.`type`)) - val objJson = platform match { - case s: DockerPlatform => encodeDockerPlatform(s) - case s: NextflowPlatform => encodeNextflowPlatform(s) - case s: NativePlatform => encodeNativePlatform(s) - } - objJson deepMerge typeJson - } + // implicit def encodePlatform[A <: Platform]: Encoder[A] = Encoder.instance { + // platform => + // val typeJson = Json.obj("type" -> Json.fromString(platform.`type`)) + // val objJson = platform match { + // case s: DockerPlatform => encodeDockerPlatform(s) + // case s: NextflowPlatform => encodeNextflowPlatform(s) + // case s: NativePlatform => encodeNativePlatform(s) + // } + // objJson deepMerge typeJson + // } implicit def decodePlatform: Decoder[Platform] = Decoder.instance { cursor => diff --git a/src/main/scala/io/viash/runners/executable/package.scala b/src/main/scala/io/viash/runners/executable/package.scala index f21dfd99a..5f0f0295e 100644 --- a/src/main/scala/io/viash/runners/executable/package.scala +++ b/src/main/scala/io/viash/runners/executable/package.scala @@ -18,7 +18,6 @@ package io.viash.runners import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object executable { import io.viash.helpers.circe._ diff --git a/src/main/scala/io/viash/runners/nextflow/package.scala b/src/main/scala/io/viash/runners/nextflow/package.scala index de6c7e70f..218e1cadb 100644 --- a/src/main/scala/io/viash/runners/nextflow/package.scala +++ b/src/main/scala/io/viash/runners/nextflow/package.scala @@ -18,7 +18,6 @@ package io.viash.runners import io.circe.{Decoder, Encoder, Json} -import io.circe.derivation.ConfiguredDecoder package object nextflow { import io.viash.helpers.circe._ diff --git a/src/main/scala/io/viash/runners/package.scala b/src/main/scala/io/viash/runners/package.scala index 591cea5de..158e0ec63 100644 --- a/src/main/scala/io/viash/runners/package.scala +++ b/src/main/scala/io/viash/runners/package.scala @@ -18,7 +18,6 @@ package io.viash import io.circe.{Decoder, Encoder, Json} -import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder} import cats.syntax.functor._ // for .widen package object runners { @@ -29,10 +28,10 @@ package object runners { import io.viash.runners.nextflow.{decodeNextflowAuto, encodeNextflowAuto} import io.viash.runners.nextflow.{decodeNextflowConfig, encodeNextflowConfig} - implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = Encoder.derived + implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = deriveConfiguredEncoder implicit val decodeExecutableRunner: Decoder[ExecutableRunner] = deriveConfiguredDecoderFullChecks - implicit val encodeNextflowRunner: Encoder.AsObject[NextflowRunner] = Encoder.derived + implicit val encodeNextflowRunner: Encoder.AsObject[NextflowRunner] = deriveConfiguredEncoder implicit val decodeNextflowRunner: Decoder[NextflowRunner] = deriveConfiguredDecoderFullChecks implicit def encodeRunner[A <: Runner]: Encoder[A] = Encoder.instance { From ef4985926a252e6a4439320840ee2fc3afa7e8e6 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Sep 2024 14:46:33 +0200 Subject: [PATCH 11/46] Tweak how the encoder & decoder placeholders are formatted --- .../helpers/circe/DeriveConfiguredDecoderFullChecks.scala | 4 +--- .../circe/DeriveConfiguredDecoderWithDeprecationCheck.scala | 3 +-- .../circe/DeriveConfiguredDecoderWithValidationCheck.scala | 3 +-- .../viash/helpers/circe/DeriveConfiguredEncoderStrict.scala | 4 +--- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala index 6dee1b41c..16107ca81 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -28,12 +28,10 @@ object DeriveConfiguredDecoderFullChecks { // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ - // def deriveConfiguredDecoderFullChecks[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A] // .validate( // validator[A], // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." // ) // .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] ) - - inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = Decoder.derivedConfigured[A] } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 78d86090e..80f0343dc 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -100,7 +100,6 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { } // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here - // def deriveConfiguredDecoderWithDeprecationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] // .prepare( checkDeprecation[A] ) - inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index af423f5ea..86d281a31 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -61,7 +61,7 @@ object DeriveConfiguredDecoderWithValidationCheck { // } // Attempts to convert the json to the desired class. Throw an exception if the conversion fails. - // def deriveConfiguredDecoderWithValidationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] // .validate( // validator[A], // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." @@ -77,7 +77,6 @@ object DeriveConfiguredDecoderWithValidationCheck { // }, // s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." // ) - inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A] def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String]): Decoder[A] = Decoder.instance { cursor => throw new ConfigParserSubTypeException(tpe, validTypes, cursor.value.toString()) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala index b299aae75..706f63c57 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala @@ -30,7 +30,7 @@ import io.viash.schemas.CollectedSchemas object DeriveConfiguredEncoderStrict { - // final def deriveConfiguredEncoderStrict[T](implicit encode: Lazy[ConfiguredAsObjectEncoder[T]], tag: TypeTag[T]) = deriveConfiguredEncoder[T] + inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredEncoder[A] // .mapJsonObject{ jsonObject => // val parameters = CollectedSchemas.getParameters[T]() // jsonObject.filterKeys( k => @@ -40,6 +40,4 @@ object DeriveConfiguredEncoderStrict { // .getOrElse(true) // fallback, shouldn't really happen // ) // } - - inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A] } From 25891ca002a3b51d5f07c6e42a8eb0cf530456cf Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Sep 2024 15:32:02 +0200 Subject: [PATCH 12/46] remove all stripMargins from field case class annotations To be solved later by either fixing the recursion that is happening or by the reflection code to call stripMargin on the fields. TBD, but the second solution could potentially cleanup some of the annotation code. --- .../scala/io/viash/config/ArgumentGroup.scala | 6 +- src/main/scala/io/viash/config/Config.scala | 72 +++++++++---------- .../scala/io/viash/config/References.scala | 2 +- .../config/arguments/BooleanArgument.scala | 28 ++++---- .../config/arguments/DoubleArgument.scala | 22 +++--- .../viash/config/arguments/FileArgument.scala | 24 +++---- .../config/arguments/IntegerArgument.scala | 22 +++--- .../viash/config/arguments/LongArgument.scala | 22 +++--- .../config/arguments/StringArgument.scala | 18 ++--- .../config/dependencies/Dependency.scala | 2 +- .../scala/io/viash/engines/DockerEngine.scala | 2 +- .../requirements/PythonRequirements.scala | 4 +- .../engines/requirements/RRequirements.scala | 4 +- .../io/viash/platforms/NextflowPlatform.scala | 10 +-- .../io/viash/runners/NextflowRunner.scala | 10 +-- .../viash/runners/nextflow/NextflowAuto.scala | 8 +-- .../runners/nextflow/NextflowConfig.scala | 12 ++-- 17 files changed, 134 insertions(+), 134 deletions(-) diff --git a/src/main/scala/io/viash/config/ArgumentGroup.scala b/src/main/scala/io/viash/config/ArgumentGroup.scala index a49ad06c3..9fd30d2ad 100644 --- a/src/main/scala/io/viash/config/ArgumentGroup.scala +++ b/src/main/scala/io/viash/config/ArgumentGroup.scala @@ -62,7 +62,7 @@ case class ArgumentGroup( @example( """description: | | A (multiline) description of the purpose of the arguments - | in this argument group.""".stripMargin, "yaml") + | in this argument group."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -76,7 +76,7 @@ case class ArgumentGroup( | - @[boolean](arg_boolean) | - @[boolean_true](arg_boolean_true) | - @[boolean_false](arg_boolean_false) - |""".stripMargin) + |"""/*.stripMargin*/) @example( """arguments: | - name: --foo @@ -91,7 +91,7 @@ case class ArgumentGroup( | multiple_sep: ";" | - name: --bar | type: string - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") arguments: List[Argument[_]] = Nil diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index 28d025b15..6319ed254 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -84,20 +84,20 @@ case class Config( @description( """A list of @[authors](author). An author must at least have a name, but can also have a list of roles, an e-mail address, and a map of custom properties. - + - +Suggested values for roles are: - + - +| Role | Abbrev. | Description | - +|------|---------|-------------| - +| maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. | - +| author | aut | for persons who have made substantial contributions to the software. | - +| contributor | ctb| for persons who have made smaller contributions (such as code patches). - +| datacontributor | dtc | for persons or organisations that contributed data sets for the software - +| copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body. - +| funder | fnd | for persons or organizations that furnished financial support for the development of the software - + - +The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive. - +""".stripMargin('+')) + | + |Suggested values for roles are: + | + || Role | Abbrev. | Description | + ||------|---------|-------------| + || maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. | + || author | aut | for persons who have made substantial contributions to the software. | + || contributor | ctb| for persons who have made smaller contributions (such as code patches). + || datacontributor | dtc | for persons or organisations that contributed data sets for the software + || copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body. + || funder | fnd | for persons or organizations that furnished financial support for the development of the software + | + |The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive. + |"""/*.stripMargin*/) @example( """authors: | - name: Jane Doe @@ -111,7 +111,7 @@ case class Config( | - name: Tim Farbe | roles: [author] | email: tim@far.be - |""".stripMargin, "yaml") + |"""/*.stripMargin*/, "yaml") @since("Viash 0.3.1") @default("Empty") authors: List[Author] = Nil, @@ -123,7 +123,7 @@ case class Config( | - `description: Description of foo`, a description of the argument group. Multiline descriptions are supported. | - `arguments: [arg1, arg2, ...]`, list of the arguments. | - |""".stripMargin) + |"""/*.stripMargin*/) @example( """argument_groups: | - name: "Input" @@ -143,7 +143,7 @@ case class Config( | - name: "--output_optional" | type: file | direction: output - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription( """component_name @@ -161,7 +161,7 @@ case class Config( | | --optional_output | type: file - |""".stripMargin, + |"""/*.stripMargin*/, "bash", "This results in the following output when calling the component with the `--help` argument:") @since("Viash 0.5.14") @@ -178,14 +178,14 @@ case class Config( | * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`. | * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`. | * is_executable: `true` / `false`, whether the resulting resource file should be made executable. - |""".stripMargin) + |"""/*.stripMargin*/) @example( """resources: | - type: r_script | path: script.R | - type: file | path: resource1.txt - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") resources: List[Resource] = Nil, @@ -206,9 +206,9 @@ case class Config( @default("Empty") @example( """description: | - + This component performs function Y and Z. - + It is possible to make this a multiline string. - +""".stripMargin('+'), + | This component performs function Y and Z. + | It is possible to make this a multiline string. + |"""/*.stripMargin*/, "yaml") description: Option[String] = None, @@ -224,7 +224,7 @@ case class Config( | - type: r_script | path: tests/test2.R | - path: resource1.txt - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") test_resources: List[Resource] = Nil, @@ -233,7 +233,7 @@ case class Config( @example( """info: | twitter: wizzkid - | classes: [ one, two, three ]""".stripMargin, "yaml") + | classes: [ one, two, three ]"""/*.stripMargin*/, "yaml") @since("Viash 0.4.0") @default("Empty") info: Json = Json.Null, @@ -247,12 +247,12 @@ case class Config( """@[Computational requirements](computational_requirements) related to running the component. |`cpus` specifies the maximum number of (logical) cpus a component is allowed to use., whereas |`memory` specifies the maximum amount of memory a component is allowed to allicate. Memory units must be - |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""".stripMargin) + |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base)."""/*.stripMargin*/) @example( """requirements: | cpus: 5 | memory: 10GB - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @since("Viash 0.6.0") @default("Empty") @@ -266,21 +266,21 @@ case class Config( | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml", "Full specification of a repository") @exampleWithDescription( """dependencies: | - name: qc/multiqc | repository: "github://openpipelines-bio/modules:0.3.0" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml", "Full specification of a repository using sugar syntax") @exampleWithDescription( """dependencies: | - name: qc/multiqc | repository: "openpipelines-bio" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml", "Reference to a repository fully specified under 'repositories'") @default("Empty") @@ -288,14 +288,14 @@ case class Config( @description( """(Pre-)defines repositories that can be used as repository in dependencies. - |Allows reusing repository definitions in case it is used in multiple dependencies.""".stripMargin) + |Allows reusing repository definitions in case it is used in multiple dependencies."""/*.stripMargin*/) @example( """repositories: | - name: openpipelines-bio | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") repositories: List[RepositoryWithName] = Nil, @@ -323,7 +323,7 @@ case class Config( | journal={Baz}, | year={2024} | } - |""".stripMargin, "yaml") + |"""/*.stripMargin*/, "yaml") @default("Empty") @since("Viash 0.9.0") references: References = References(), @@ -336,7 +336,7 @@ case class Config( | homepage: "https://viash.io" | documentation: "https://viash.io/reference/" | issue_tracker: "https://github.com/viash-io/viash/issues" - |""".stripMargin, "yaml") + |"""/*.stripMargin*/, "yaml") @default("Empty") @since("Viash 0.9.0") links: Links = Links(), @@ -353,7 +353,7 @@ case class Config( | | - @[ExecutableRunner](executable_runner) | - @[NextflowRunner](nextflow_runner) - |""".stripMargin) + |"""/*.stripMargin*/) @since("Viash 0.8.0") @default("Empty") runners: List[Runner] = Nil, @@ -362,7 +362,7 @@ case class Config( | | - @[NativeEngine](native_engine) | - @[DockerEngine](docker_engine) - |""".stripMargin) + |"""/*.stripMargin*/) @since("Viash 0.8.0") @default("Empty") engines: List[Engine] = Nil, diff --git a/src/main/scala/io/viash/config/References.scala b/src/main/scala/io/viash/config/References.scala index b24ee9d8d..ab63add56 100644 --- a/src/main/scala/io/viash/config/References.scala +++ b/src/main/scala/io/viash/config/References.scala @@ -48,7 +48,7 @@ case class References( | journal={Baz}, | year={2024} | } - |""".stripMargin, "yaml") + |"""/*.stripMargin*/, "yaml") @default("Empty") bibtex: OneOrMore[String] = Nil, ) diff --git a/src/main/scala/io/viash/config/arguments/BooleanArgument.scala b/src/main/scala/io/viash/config/arguments/BooleanArgument.scala index 90152a27a..9835de12c 100644 --- a/src/main/scala/io/viash/config/arguments/BooleanArgument.scala +++ b/src/main/scala/io/viash/config/arguments/BooleanArgument.scala @@ -43,7 +43,7 @@ case class BooleanArgument( | - `--trim` is a long option, which can be passed with `executable_name --trim` | - `-t` is a short option, which can be passed with `executable_name -t` | - `trim` is an argument, which can be passed with `executable_name trim` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -66,7 +66,7 @@ case class BooleanArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -74,7 +74,7 @@ case class BooleanArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -84,7 +84,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | example: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") example: OneOrMore[Boolean] = Nil, @@ -94,7 +94,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | default: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") default: OneOrMore[Boolean] = Nil, @@ -104,7 +104,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | required: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("False") required: Boolean = false, @@ -117,7 +117,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | multiple: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_boolean=true:true:false", "bash", "Here's an example of how to use this:") @default("False") @@ -129,7 +129,7 @@ case class BooleanArgument( | type: boolean | multiple: true | multiple_sep: ";" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_boolean=true,true,false", "bash", "Here's an example of how to use this:") @default(";") @@ -179,7 +179,7 @@ case class BooleanTrueArgument( | - `--silent` is a long option, which can be passed with `executable_name --silent` | - `-s` is a short option, which can be passed with `executable_name -s` | - `silent` is an argument, which can be passed with `executable_name silent` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -202,7 +202,7 @@ case class BooleanTrueArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -210,7 +210,7 @@ case class BooleanTrueArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -271,7 +271,7 @@ case class BooleanFalseArgument( | - `--no-log` is a long option, which can be passed with `executable_name --no-log` | - `-n` is a short option, which can be passed with `executable_name -n` | - `no-log` is an argument, which can be passed with `executable_name no-log` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -294,7 +294,7 @@ case class BooleanFalseArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -302,7 +302,7 @@ case class BooleanFalseArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, diff --git a/src/main/scala/io/viash/config/arguments/DoubleArgument.scala b/src/main/scala/io/viash/config/arguments/DoubleArgument.scala index 3448b1649..273818e83 100644 --- a/src/main/scala/io/viash/config/arguments/DoubleArgument.scala +++ b/src/main/scala/io/viash/config/arguments/DoubleArgument.scala @@ -39,7 +39,7 @@ case class DoubleArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -62,7 +62,7 @@ case class DoubleArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -70,7 +70,7 @@ case class DoubleArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -80,7 +80,7 @@ case class DoubleArgument( """- name: --my_double | type: double | example: 5.8 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") example: OneOrMore[Double] = Nil, @@ -90,7 +90,7 @@ case class DoubleArgument( """- name: --my_double | type: double | default: 5.8 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") default: OneOrMore[Double] = Nil, @@ -100,7 +100,7 @@ case class DoubleArgument( """- name: --my_double | type: double | required: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("False") required: Boolean = false, @@ -110,7 +110,7 @@ case class DoubleArgument( """- name: --my_double | type: double | min: 25.5 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") min: Option[Double] = None, @@ -119,8 +119,8 @@ case class DoubleArgument( """- name: --my_double | type: double | max: 80.4 - |""".stripMargin, - "yaml") + |"""/*.stripMargin*/, + "yaml") max: Option[Double] = None, @undocumented @@ -131,7 +131,7 @@ case class DoubleArgument( """- name: --my_double | type: double | multiple: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_double=5.8:22.6:200.4", "bash", "Here's an example of how to use this:") @default("False") @@ -143,7 +143,7 @@ case class DoubleArgument( | type: double | multiple: true | multiple_sep: ";" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_double=5.8,22.6,200.4", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/FileArgument.scala b/src/main/scala/io/viash/config/arguments/FileArgument.scala index 02660fcef..6216d80b2 100644 --- a/src/main/scala/io/viash/config/arguments/FileArgument.scala +++ b/src/main/scala/io/viash/config/arguments/FileArgument.scala @@ -40,7 +40,7 @@ case class FileArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -63,7 +63,7 @@ case class FileArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -71,7 +71,7 @@ case class FileArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -81,7 +81,7 @@ case class FileArgument( """- name: --my_file | type: file | example: data.csv - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") example: OneOrMore[Path] = Nil, @@ -91,7 +91,7 @@ case class FileArgument( """- name: --my_file | type: file | default: data.csv - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") default: OneOrMore[Path] = Nil, @@ -102,7 +102,7 @@ case class FileArgument( """- name: --my_file | type: file | must_exist: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("True") must_exist: Boolean = true, @@ -113,7 +113,7 @@ case class FileArgument( | type: file | direction: output | create_parent: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("True") create_parent: Boolean = true, @@ -123,7 +123,7 @@ case class FileArgument( """- name: --my_file | type: file | required: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("False") required: Boolean = false, @@ -133,7 +133,7 @@ case class FileArgument( """- name: --my_output_file | type: file | direction: output - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Input") direction: Direction = Input, @@ -152,12 +152,12 @@ case class FileArgument( | automatically attempt to expand the expression. | |Other output arguments (e.g. integer, double, ...) are not supported yet. - |""".stripMargin) + |"""/*.stripMargin*/) @example( """- name: --my_files | type: file | multiple: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_files=firstFile.csv:anotherFile.csv:yetAnother.csv", "bash", "Here's an example of how to use this:") @default("False") @@ -169,7 +169,7 @@ case class FileArgument( | type: file | multiple: true | multiple_sep: ";" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_files=firstFile.csv,anotherFile.csv,yetAnother.csv", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/IntegerArgument.scala b/src/main/scala/io/viash/config/arguments/IntegerArgument.scala index 3895c8925..548d51cb8 100644 --- a/src/main/scala/io/viash/config/arguments/IntegerArgument.scala +++ b/src/main/scala/io/viash/config/arguments/IntegerArgument.scala @@ -39,7 +39,7 @@ case class IntegerArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -62,7 +62,7 @@ case class IntegerArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -70,7 +70,7 @@ case class IntegerArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -80,7 +80,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | example: 100 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") example: OneOrMore[Int] = Nil, @@ -90,7 +90,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | default: 100 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") default: OneOrMore[Int] = Nil, @@ -100,7 +100,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | required: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("False") required: Boolean = false, @@ -110,7 +110,7 @@ case class IntegerArgument( """- name: --values | type: integer | choices: [1024, 2048, 4096] - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") choices: List[Int] = Nil, @@ -120,7 +120,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | min: 50 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") min: Option[Int] = None, @@ -129,7 +129,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | max: 150 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") max: Option[Int] = None, @@ -141,7 +141,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | multiple: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:") @default("False") @@ -153,7 +153,7 @@ case class IntegerArgument( | type: integer | multiple: true | multiple_sep: ";" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/LongArgument.scala b/src/main/scala/io/viash/config/arguments/LongArgument.scala index 075b9e1f0..92a7ffb2f 100644 --- a/src/main/scala/io/viash/config/arguments/LongArgument.scala +++ b/src/main/scala/io/viash/config/arguments/LongArgument.scala @@ -40,7 +40,7 @@ case class LongArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -63,7 +63,7 @@ case class LongArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -71,7 +71,7 @@ case class LongArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -81,7 +81,7 @@ case class LongArgument( """- name: --my_long | type: long | example: 100 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") example: OneOrMore[Long] = Nil, @@ -91,7 +91,7 @@ case class LongArgument( """- name: --my_long | type: long | default: 100 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") default: OneOrMore[Long] = Nil, @@ -101,7 +101,7 @@ case class LongArgument( """- name: --my_long | type: long | required: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("False") required: Boolean = false, @@ -111,7 +111,7 @@ case class LongArgument( """- name: --values | type: long | choices: [1024, 2048, 4096] - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") choices: List[Long] = Nil, @@ -121,7 +121,7 @@ case class LongArgument( """- name: --my_long | type: long | min: 50 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") min: Option[Long] = None, @@ -130,7 +130,7 @@ case class LongArgument( """- name: --my_long | type: long | max: 150 - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") max: Option[Long] = None, @@ -142,7 +142,7 @@ case class LongArgument( """- name: --my_long | type: long | multiple: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:") @default("False") @@ -154,7 +154,7 @@ case class LongArgument( | type: long | multiple: true | multiple_sep: ";" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/StringArgument.scala b/src/main/scala/io/viash/config/arguments/StringArgument.scala index 1a94f7a54..133115990 100644 --- a/src/main/scala/io/viash/config/arguments/StringArgument.scala +++ b/src/main/scala/io/viash/config/arguments/StringArgument.scala @@ -39,7 +39,7 @@ case class StringArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |""".stripMargin) + |"""/*.stripMargin*/) name: String, @description("List of alternative format variations for this argument.") @@ -62,7 +62,7 @@ case class StringArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument.""".stripMargin, "yaml") + | this argument."""/*.stripMargin*/, "yaml") @default("Empty") description: Option[String] = None, @@ -70,7 +70,7 @@ case class StringArgument( @example( """info: | category: cat1 - | labels: [one, two, three]""".stripMargin, "yaml") + | labels: [one, two, three]"""/*.stripMargin*/, "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -80,7 +80,7 @@ case class StringArgument( """- name: --my_string | type: string | example: "Hello World" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") example: OneOrMore[String] = Nil, @@ -90,7 +90,7 @@ case class StringArgument( """- name: --my_string | type: string | default: "The answer is 42" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") default: OneOrMore[String] = Nil, @@ -100,7 +100,7 @@ case class StringArgument( """- name: --my_string | type: string | required: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") required: Boolean = false, @@ -110,7 +110,7 @@ case class StringArgument( """- name: --language | type: string | choices: ["python", "r", "javascript"] - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @default("Empty") choices: List[String] = Nil, @@ -123,7 +123,7 @@ case class StringArgument( """- name: --my_string | type: string | multiple: true - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_string=Marc:Susan:Paul", "bash", "Here's an example of how to use this:") @default("False") @@ -135,7 +135,7 @@ case class StringArgument( | type: string | multiple: true | multiple_sep: ";" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @exampleWithDescription("my_component --my_string=Marc,Susan,Paul", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/dependencies/Dependency.scala b/src/main/scala/io/viash/config/dependencies/Dependency.scala index e5b03e338..b1ea2f671 100644 --- a/src/main/scala/io/viash/config/dependencies/Dependency.scala +++ b/src/main/scala/io/viash/config/dependencies/Dependency.scala @@ -76,7 +76,7 @@ case class Dependency( |This must either be a full definition of the repository or the name of a repository referenced as it is defined under repositories. |Additionally, the full definition can be specified as a single string where all parameters such as repository type, url, branch or tag are specified. |Omitting the value sets the dependency as a local dependency, ie. the dependency is available in the same namespace as the component. - |""".stripMargin) + |"""/*.stripMargin*/) @default("Empty") repository: Either[String, Repository] = Right(LocalRepository()), diff --git a/src/main/scala/io/viash/engines/DockerEngine.scala b/src/main/scala/io/viash/engines/DockerEngine.scala index bf17cbe64..ba4fced3e 100644 --- a/src/main/scala/io/viash/engines/DockerEngine.scala +++ b/src/main/scala/io/viash/engines/DockerEngine.scala @@ -109,7 +109,7 @@ final case class DockerEngine( | - @[yum](yum_req) | |The order in which these dependencies are specified determines the order in which they will be installed. - |""".stripMargin) + |"""/*.stripMargin*/) @default("Empty") setup: List[Requirements] = Nil, diff --git a/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala b/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala index f96ed4f93..be7f1c3d0 100644 --- a/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala @@ -87,8 +87,8 @@ case class PythonRequirements( @description("Specifies a code block to run as part of the build.") @example("""script: | - # print("Running custom code") - # x = 1 + 1 == 2""".stripMargin('#'), "yaml") + | print("Running custom code") + | x = 1 + 1 == 2"""/*.stripMargin*/, "yaml") @default("Empty") script: OneOrMore[String] = Nil, diff --git a/src/main/scala/io/viash/engines/requirements/RRequirements.scala b/src/main/scala/io/viash/engines/requirements/RRequirements.scala index f306099d2..b1e31b3c4 100644 --- a/src/main/scala/io/viash/engines/requirements/RRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/RRequirements.scala @@ -78,8 +78,8 @@ case class RRequirements( @description("Specifies a code block to run as part of the build.") @example("""script: | - # cat("Running custom code\n") - # install.packages("anndata")""".stripMargin('#'), "yaml") + | cat("Running custom code\n") + | install.packages("anndata")"""/*.stripMargin*/, "yaml") @default("Empty") script: OneOrMore[String] = Nil, diff --git a/src/main/scala/io/viash/platforms/NextflowPlatform.scala b/src/main/scala/io/viash/platforms/NextflowPlatform.scala index 5df75c206..1de8ba6cc 100644 --- a/src/main/scala/io/viash/platforms/NextflowPlatform.scala +++ b/src/main/scala/io/viash/platforms/NextflowPlatform.scala @@ -45,13 +45,13 @@ case class NextflowPlatform( // nxf params @description( """@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts. - |""".stripMargin) + |"""/*.stripMargin*/) @example( """directives: | container: rocker/r-ver:4.1 | label: highcpu | cpus: 4 - | memory: 16 GB""".stripMargin, + | memory: 16 GB"""/*.stripMargin*/, "yaml") @default("Empty") directives: NextflowDirectives = NextflowDirectives(), @@ -66,17 +66,17 @@ case class NextflowPlatform( || `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` | || `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. If equal to `"state"`, also a `.state.yaml` file will be published in the publish dir. Will throw an error if `params.publishDir` is not defined. | `false` | | - |""".stripMargin) + |"""/*.stripMargin*/) @example( """auto: - | publish: true""".stripMargin, + | publish: true"""/*.stripMargin*/, "yaml") @default( """simplifyInput: true |simplifyOutput: false |transcript: false |publish: false - |""".stripMargin) + |"""/*.stripMargin*/) auto: NextflowAuto = NextflowAuto(), @description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.") diff --git a/src/main/scala/io/viash/runners/NextflowRunner.scala b/src/main/scala/io/viash/runners/NextflowRunner.scala index 90df9416b..37e617c48 100644 --- a/src/main/scala/io/viash/runners/NextflowRunner.scala +++ b/src/main/scala/io/viash/runners/NextflowRunner.scala @@ -53,13 +53,13 @@ final case class NextflowRunner( // nxf params @description( """@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts. - |""".stripMargin) + |"""/*.stripMargin*/) @example( """directives: | container: rocker/r-ver:4.1 | label: highcpu | cpus: 4 - | memory: 16 GB""".stripMargin, + | memory: 16 GB"""/*.stripMargin*/, "yaml") @default("Empty") directives: NextflowDirectives = NextflowDirectives(), @@ -74,17 +74,17 @@ final case class NextflowRunner( || `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` | || `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. If equal to `"state"`, also a `.state.yaml` file will be published in the publish dir. Will throw an error if `params.publishDir` is not defined. | `false` | | - |""".stripMargin) + |"""/*.stripMargin*/) @example( """auto: - | publish: true""".stripMargin, + | publish: true"""/*.stripMargin*/, "yaml") @default( """simplifyInput: true |simplifyOutput: false |transcript: false |publish: false - |""".stripMargin) + |"""/*.stripMargin*/) auto: NextflowAuto = NextflowAuto(), @description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.") diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala b/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala index 813309063..6451b3d59 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala @@ -25,7 +25,7 @@ case class NextflowAuto( """If `true`, an input tuple only containing only a single File (e.g. `["foo", file("in.h5ad")]`) is automatically transformed to a map (i.e. `["foo", [ input: file("in.h5ad") ] ]`). | |Default: `true`. - |""".stripMargin) + |"""/*.stripMargin*/) @default("True") simplifyInput: Boolean = true, @@ -33,7 +33,7 @@ case class NextflowAuto( """If `true`, an output tuple containing a map with a File (e.g. `["foo", [ output: file("out.h5ad") ] ]`) is automatically transformed to a map (i.e. `["foo", file("out.h5ad")]`). | |Default: `false`. - |""".stripMargin) + |"""/*.stripMargin*/) @default("False") simplifyOutput: Boolean = false, @@ -43,7 +43,7 @@ case class NextflowAuto( |Will throw an error if neither are defined. | |Default: `false`. - |""".stripMargin) + |"""/*.stripMargin*/) @default("False") transcript: Boolean = false, @@ -53,7 +53,7 @@ case class NextflowAuto( |Will throw an error if `params.publishDir` is not defined. | |Default: `false`. - |""".stripMargin) + |"""/*.stripMargin*/) @default("False") publish: Either[Boolean, String] = Left(false) ) { diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala index e80ae101d..7018b27e6 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala @@ -33,7 +33,7 @@ case class NextflowConfig( | |Conceptually it is possible for a Viash Config to overwrite the full labels parameter, however likely it is more efficient to add additional labels |in the Viash Package with a config mod. - |""".stripMargin) + |"""/*.stripMargin*/) @exampleWithDescription( """labels: | lowmem: "memory = 4.GB" @@ -44,7 +44,7 @@ case class NextflowConfig( | highcpu: "cpus = 20" | vhighmem: "memory = 100.GB" | vhighcpu: "cpus = 40" - |""".stripMargin, + |"""/*.stripMargin*/, "yaml", "Replace the default labels with a different set of labels") @exampleWithDescription( @@ -55,14 +55,14 @@ case class NextflowConfig( """config_mods: | | .runners[.type == "nextflow"].config.labels.lowmem := "memory = 4.GB" | .runners[.type == "nextflow"].config.labels.lowcpu := "cpus = 4" - |""".stripMargin, + |"""/*.stripMargin*/, "viash_package_file", "Add 'lowmem' and 'lowcpu' to the default labels by using the Viash Package file" ) @exampleWithDescription( """config_mods: | | .runners[.type == "nextflow"].config.labels := { lowmem: "memory = 4.GB", lowcpu: "cpus = 4", midmem: "memory = 25.GB", midcpu: "cpus = 10", highmem: "memory = 50.GB", highcpu: "cpus = 20", vhighmem: "memory = 100.GB", vhighcpu: "cpus = 40" } - |""".stripMargin, + |"""/*.stripMargin*/, "viash_package_file", "Replace the default labels with a different set of labels by using the Viash Package file" ) @@ -90,14 +90,14 @@ case class NextflowConfig( @description( """Includes a single string or list of strings into the nextflow.config file. |This can be used to add custom profiles or include an additional config file. - |""".stripMargin) + |"""/*.stripMargin*/) @example( """script: | - | | profiles { | ... | } - |""".stripMargin, + |"""/*.stripMargin*/, "yaml") @example("""script: includeConfig("config.config")""", "yaml") @default("Empty") From 9ed9d94c02ab4df757cfdd015e067bd335458a01 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 12 Sep 2024 13:45:52 +0200 Subject: [PATCH 13/46] Add custom encoder & decoder for Status enum Manually written code, I bet there is a better way --- src/main/scala/io/viash/config/package.scala | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index 63690be5a..e89736506 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -27,6 +27,8 @@ import config.Links import config.References import config.Status._ import config.arguments._ +import io.circe.DecodingFailure +import io.circe.DecodingFailure.Reason.CustomReason package object config { import io.viash.helpers.circe._ @@ -286,10 +288,21 @@ package object config { implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks // encoder and decoder for Status, make string lowercase before decoding - implicit val encodeStatus: Encoder[Status] = deriveConfiguredEncoder - implicit val decodeStatus: Decoder[Status] = deriveConfiguredDecoderFullChecks[Status].prepare { - _.withFocus(_.mapString(_.toLowerCase())) + implicit val encodeStatus: Encoder[Status] = Encoder.instance{ + case Enabled => Json.fromString("enabled") + case Disabled => Json.fromString("disabled") + case Deprecated => Json.fromString("deprecated") } + implicit val decodeStatus: Decoder[Status] = Decoder.instance(cursor => + for { + str <- cursor.as[String] + status <- str.toLowerCase() match + case "enabled" => Right(Enabled) + case "disabled" => Right(Disabled) + case "deprecated" => Right(Deprecated) + case other => Left(DecodingFailure(CustomReason(s"$other is not a valid Status"), cursor)) + } yield status + ) implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks From b299ac98d67ce13312df28bb80b6c1c34455b28a Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 12 Sep 2024 13:48:28 +0200 Subject: [PATCH 14/46] Add back some validation code to assist with invalid fields Temporary way of getting the class name Not fetching the valid fields yet --- .../DeriveConfiguredDecoderFullChecks.scala | 17 ++--- ...onfiguredDecoderWithDeprecationCheck.scala | 2 +- ...ConfiguredDecoderWithValidationCheck.scala | 73 +++++++++---------- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala index 16107ca81..d11c278f7 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -17,21 +17,18 @@ package io.viash.helpers.circe -// import shapeless.Lazy -// import scala.reflect.runtime.universe._ - import io.circe.Decoder import io.circe.derivation.Configuration import scala.deriving.Mirror object DeriveConfiguredDecoderFullChecks { - // import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ - // import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation + import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck.validator inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A] - // .validate( - // validator[A], - // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." - // ) - // .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] ) + .validate( + validator[A], + s"Could not convert json to ${A.getClass().getName().stripSuffix("$")}." + ) + .prepare( checkDeprecation[A] ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 80f0343dc..aac6e99d1 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -101,5 +101,5 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] - // .prepare( checkDeprecation[A] ) + .prepare( checkDeprecation[A] ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 86d281a31..7cffb6a60 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -31,54 +31,51 @@ object DeriveConfiguredDecoderWithValidationCheck { // Validate the json can correctly converted to the required type by actually converting it. // Throw an exception when the conversion fails. - // def validator[A](pred: HCursor)(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Boolean = { - // val d = deriveConfiguredDecoder[A] - // val v = d(pred) + inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A]): Boolean = { + val d = deriveConfiguredDecoder[A] + val v = d(pred) - // v.fold(error => { - // val usedFields = pred.value.asObject.map(_.keys.toSeq) - // val validFields = typeOf[A].members.filter(m => !m.isMethod).map(_.name.toString.strip()).toSeq - // val invalidFields = usedFields.map(_.diff(validFields)) + v.fold(error => { + val usedFields = pred.value.asObject.map(_.keys.toSeq) + val validFields = Seq.empty //typeOf[A].members.filter(m => !m.isMethod).map(_.name.toString.strip()).toSeq + val invalidFields = usedFields.map(_.diff(validFields)) - // val fieldsHint = invalidFields match { - // case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}") - // case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}") - // case _ => None - // } + val fieldsHint = invalidFields match { + case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}") + case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}") + case _ => None + } - // val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") + val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - // val hint = (fieldsHint, historyString, error.message) match { - // case (Some(a), h, _) if h != "" => Some(s".$h -> $a") - // case (Some(a), _, _) => Some(a) - // case (None, h, m) if h != "" => Some(s".$h -> $m") - // case _ => None - // } + val hint = (fieldsHint, historyString, error.message) match { + case (Some(a), h, _) if h != "" => Some(s".$h -> $a") + case (Some(a), _, _) => Some(a) + case (None, h, m) if h != "" => Some(s".$h -> $m") + case _ => None + } - // throw new ConfigParserValidationException(typeOf[A].baseClasses.head.fullName, pred.value.toString(), hint) - // false - // }, _ => true) - // } + throw new ConfigParserValidationException(A.getClass().getName().stripSuffix("$") /* typeOf[A].baseClasses.head.fullName */, pred.value.toString(), hint) + false + }, _ => true) + } // Attempts to convert the json to the desired class. Throw an exception if the conversion fails. inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] - // .validate( - // validator[A], - // s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." - // ) + .validate( + validator[A], + s"Could not convert json to ${A.getClass().getName().stripSuffix("$") /* typeOf[A].baseClasses.head.fullName */}." + ) // Dummy decoder to generate exceptions when an invalid type is specified // We need a valid class type to be specified - // def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] - // .validate( - // pred => { - // throw new ConfigParserSubTypeException(tpe, validTypes, pred.value.toString()) - // false - // }, - // s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." - // ) - def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String]): Decoder[A] = Decoder.instance { cursor => - throw new ConfigParserSubTypeException(tpe, validTypes, cursor.value.toString()) - } + inline def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A] + .validate( + pred => { + throw new ConfigParserSubTypeException(tpe, validTypes, pred.value.toString()) + false + }, + s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." + ) } From 3e9da3901cb1397cd623fd26210dbd1e0eaed502 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 12 Sep 2024 15:20:06 +0200 Subject: [PATCH 15/46] provide safeguard to solve empty packageconfig issues --- .../DeriveConfiguredDecoderWithValidationCheck.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 7cffb6a60..dfd434559 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -31,9 +31,15 @@ object DeriveConfiguredDecoderWithValidationCheck { // Validate the json can correctly converted to the required type by actually converting it. // Throw an exception when the conversion fails. - inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A]): Boolean = { + inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A], inline configuration: Configuration): Boolean = { val d = deriveConfiguredDecoder[A] - val v = d(pred) + // val v = d(pred) + // TODO not entirely sure why this is needed instead of just doing `val v = d(pred)` + // goes wrong when decoding empty PackageConfig + val v = pred match { + case pred if pred.value.isNull => Right(null.asInstanceOf[A]) + case _ => d(pred) + } v.fold(error => { val usedFields = pred.value.asObject.map(_.keys.toSeq) From 83aaaee9533ae3e277d5cda09090eca694cc7c0b Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 12 Sep 2024 20:16:37 +0200 Subject: [PATCH 16/46] Start using reflection to get class name and fields Mark regex issues as unchecked to be solved later --- .../scala/io/viash/config/ConfigMeta.scala | 2 +- .../io/viash/config/arguments/Argument.scala | 2 +- .../scala/io/viash/helpers/Mirroring.scala | 34 +++++++++++++++++++ .../DeriveConfiguredDecoderFullChecks.scala | 3 +- ...ConfiguredDecoderWithValidationCheck.scala | 7 ++-- .../runners/nextflow/NextflowTestHelper.scala | 2 +- 6 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 src/main/scala/io/viash/helpers/Mirroring.scala diff --git a/src/main/scala/io/viash/config/ConfigMeta.scala b/src/main/scala/io/viash/config/ConfigMeta.scala index 63a56cedb..38aa24ea8 100644 --- a/src/main/scala/io/viash/config/ConfigMeta.scala +++ b/src/main/scala/io/viash/config/ConfigMeta.scala @@ -118,7 +118,7 @@ object ConfigMeta { val configYamlStr2 = placeholderMap.foldLeft(configYamlStr) { case (configStr, (res, placeholder)) => val IndentRegex = ("( *)text: \"" + placeholder + "\"").r - val IndentRegex(indent) = IndentRegex.findFirstIn(configStr).getOrElse("") + val IndentRegex(indent) = IndentRegex.findFirstIn(configStr).getOrElse("") : @unchecked configStr.replace( "\"" + placeholder + "\"", "|\n" + indent + " " + res.text.get.replace("\n", "\n " + indent) + "\n" diff --git a/src/main/scala/io/viash/config/arguments/Argument.scala b/src/main/scala/io/viash/config/arguments/Argument.scala index 98a528a00..869da0e33 100644 --- a/src/main/scala/io/viash/config/arguments/Argument.scala +++ b/src/main/scala/io/viash/config/arguments/Argument.scala @@ -77,7 +77,7 @@ abstract class Argument[Type] { val dest: String private val pattern = "^(-*)(.*)$".r - val pattern(flags, plainName) = name + val pattern(flags, plainName) = name : @unchecked /** Common parameter name for this argument */ val par: String = dest + "_" + plainName diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala new file mode 100644 index 000000000..97a7e9e92 --- /dev/null +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers + +import scala.quoted.* + +inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } +inline def niceNameOf[T]: String = ${ niceNameImpl[T] } + +def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = + import quotes.reflect.* + val cls = TypeRepr.of[T].classSymbol.get + val fieldSymbols = cls.caseFields.map(_.name) + Expr(fieldSymbols) + +def niceNameImpl[T: Type](using Quotes): Expr[String] = + import quotes.reflect.* + val name = TypeRepr.of[T].typeSymbol.name + Expr(name) \ No newline at end of file diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala index d11c278f7..20149f631 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -20,6 +20,7 @@ package io.viash.helpers.circe import io.circe.Decoder import io.circe.derivation.Configuration import scala.deriving.Mirror +import io.viash.helpers.niceNameOf object DeriveConfiguredDecoderFullChecks { import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation @@ -28,7 +29,7 @@ object DeriveConfiguredDecoderFullChecks { inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A] .validate( validator[A], - s"Could not convert json to ${A.getClass().getName().stripSuffix("$")}." + s"Could not convert json to ${niceNameOf[A]}." ) .prepare( checkDeprecation[A] ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index dfd434559..6e78db2e2 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -26,6 +26,7 @@ import io.circe.ACursor import io.viash.exceptions.ConfigParserSubTypeException import io.viash.exceptions.ConfigParserValidationException import io.circe.HCursor +import io.viash.helpers.{niceNameOf, fieldsOf} object DeriveConfiguredDecoderWithValidationCheck { @@ -43,7 +44,7 @@ object DeriveConfiguredDecoderWithValidationCheck { v.fold(error => { val usedFields = pred.value.asObject.map(_.keys.toSeq) - val validFields = Seq.empty //typeOf[A].members.filter(m => !m.isMethod).map(_.name.toString.strip()).toSeq + val validFields = fieldsOf[A] val invalidFields = usedFields.map(_.diff(validFields)) val fieldsHint = invalidFields match { @@ -61,7 +62,7 @@ object DeriveConfiguredDecoderWithValidationCheck { case _ => None } - throw new ConfigParserValidationException(A.getClass().getName().stripSuffix("$") /* typeOf[A].baseClasses.head.fullName */, pred.value.toString(), hint) + throw new ConfigParserValidationException(niceNameOf[A], pred.value.toString(), hint) false }, _ => true) } @@ -70,7 +71,7 @@ object DeriveConfiguredDecoderWithValidationCheck { inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] .validate( validator[A], - s"Could not convert json to ${A.getClass().getName().stripSuffix("$") /* typeOf[A].baseClasses.head.fullName */}." + s"Could not convert json to ${niceNameOf[A]}." ) // Dummy decoder to generate exceptions when an invalid type is specified diff --git a/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala b/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala index 0c5e75b73..6565630e6 100644 --- a/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala +++ b/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala @@ -22,7 +22,7 @@ object NextflowTestHelper { val lines = output.split("\n").find(DebugRegex.findFirstIn(_).isDefined) assert(lines.isDefined) - val DebugRegex(path) = lines.get + val DebugRegex(path) = lines.get : @unchecked val src = Source.fromFile(path) try { From 8d422d08a55b2e2812c41d4ae70f067e7640b6b9 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 13 Sep 2024 08:27:56 +0200 Subject: [PATCH 17/46] reactivate some code --- .../io/viash/schemas/CollectedSchemas.scala | 35 ++- .../scala/io/viash/schemas/JsonSchema.scala | 245 +++++++++--------- 2 files changed, 139 insertions(+), 141 deletions(-) diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 93376ca1a..a97755a81 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -234,31 +234,30 @@ object CollectedSchemas { def getJson: Json = data.asJson - // private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = { - // val issueMembers = members - // .toList - // .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement. - // .map{ case (k, v) => (k, v.map(_.symbol.annotations.length).sum) } // (name, # annotations) - // .filter(_._2 == 0) - // .map(_._1) - - // val ownClassArr = if (classes.head.annotations.length == 0) Seq("__this__") else Nil - // issueMembers ++ ownClassArr - // } + private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = { + // val issueMembers = members + // .toList + // .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement. + // .map{ case (k, v) => (k, v.map(_.symbol.annotations.length).sum) } // (name, # annotations) + // .filter(_._2 == 0) + // .map(_._1) + + // val ownClassArr = if (classes.head.annotations.length == 0) Seq("__this__") else Nil + // issueMembers ++ ownClassArr + Nil + } def getMemberName(members: Map[String,List[MemberInfo]], classes: List[Symbol]): String = classes.head.shortName // Main call for checking whether all arguments are annotated // Add extra non-annotated value so we can always somewhat check the code is functional - // def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap { - // v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) - // }.toMap - def getAllNonAnnotated: Map[String, String] = Map.empty + def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap { + v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) + }.toMap def getAllDeprecations: Map[String, DeprecatedOrRemovedSchema] = { - // val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap - // arr.filter(t => t._2.isDefined).map(t => (t._1, t._2.get)) - Map.empty + val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap + arr.filter(t => t._2.isDefined).map(t => (t._1, t._2.get)) } } diff --git a/src/main/scala/io/viash/schemas/JsonSchema.scala b/src/main/scala/io/viash/schemas/JsonSchema.scala index 8b7ff13dd..eab53af77 100644 --- a/src/main/scala/io/viash/schemas/JsonSchema.scala +++ b/src/main/scala/io/viash/schemas/JsonSchema.scala @@ -119,130 +119,129 @@ object JsonSchema { data.find(_.name == "__this__").get def createSchema(info: List[ParameterSchema])(implicit config: SchemaConfig): (String, Json) = { - ("", Json.Null) - - // def removeMarkup(text: String): String = { - // val markupRegex = raw"@\[(.*?)\]\(.*?\)".r - // val backtickRegex = "`(\"[^`\"]*?\")`".r - // val textWithoutMarkup = markupRegex.replaceAllIn(text, "$1") - // backtickRegex.replaceAllIn(textWithoutMarkup, "$1") - // } - - // val thisParameter = getThisParameter(info) - // val description = removeMarkup(thisParameter.description.get) - // val subclass = thisParameter.subclass.map(l => l.head) - // val properties = - // info - // .filter(p => !p.name.startsWith("__")) // remove __this__ - // .filter(p => !p.removed.isDefined && (!config.strict || !p.deprecated.isDefined)) // always remove 'removed' arguments and if need be create a strict schema by removing deprecated - // .filter(p => !config.strict || !(p.name == "arguments" && (thisParameter.`type` == "Functionality" || thisParameter.`type` == "Config"))) // exception: remove 'arguments' in 'Functionality' for strict schema - // val propertiesJson = properties.map(p => { - // val pDescription = p.description.map(s => removeMarkup(s)) - // val trimmedType = p.`type` match { - // case s if s.startsWith("Option[") => s.stripPrefix("Option[").stripSuffix("]") - // case s => s - // } - - // val mapRegex = "(List)?Map\\[String,(\\w*)\\]".r - - // implicit val useAllInEither = thisParameter.`type` == "NextflowDirectives" || thisParameter.`type` == "NextflowAuto" - - // trimmedType match { - // case s"List[$s]" => - // (p.name, arrayType(s, pDescription)) - - // case "Either[String,List[String]]" => - // (p.name, eitherJson( - // valueType("String", pDescription), - // arrayType("String", pDescription) - // )) - - // case "Either[Map[String,String],String]" => - // (p.name, eitherJson( - // mapType("String", pDescription), - // valueType("String", pDescription) - // )) - - // case s"Either[$s,$t]" => - // (p.name, eitherJson( - // valueType(s, pDescription), - // valueType(t, pDescription) - // )) - - // case "OneOrMore[Map[String,String]]" => - // (p.name, oneOrMoreJson( - // mapType("String", pDescription) - // )) - - // case "OneOrMore[Either[String,Map[String,String]]]" => - // (p.name, oneOrMoreJson( - // eitherJson( - // valueType("String", pDescription), - // mapType("String", pDescription) - // ) - // )) - - // case s"OneOrMore[$s]" => - // if (s == "String" && p.name == "port" && subclass == Some("executable")) { - // // Custom exception - // // This is the port field for a excutable runner. - // // We want to allow a Strings or Ints. - // (p.name, eitherJson( - // valueType("Int", pDescription), - // valueType("String", pDescription), - // arrayType("Int", pDescription), - // arrayType("String", pDescription) - // )) - // } else { - // (p.name, oneOrMoreType(s, pDescription)) - // } - - // case mapRegex(_, s) => - // (p.name, mapType(s, pDescription)) - - // case s if p.name == "type" && subclass.isDefined => - // var subclassString = subclass.get.stripSuffix("withname") - // if (config.minimal) { - // ("type", Json.obj( - // "const" -> Json.fromString(subclassString) - // )) - // } else { - // ("type", Json.obj( - // "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class - // "const" -> Json.fromString(subclassString) - // )) - // } + + def removeMarkup(text: String): String = { + val markupRegex = raw"@\[(.*?)\]\(.*?\)".r + val backtickRegex = "`(\"[^`\"]*?\")`".r + val textWithoutMarkup = markupRegex.replaceAllIn(text, "$1") + backtickRegex.replaceAllIn(textWithoutMarkup, "$1") + } + + val thisParameter = getThisParameter(info) + val description = removeMarkup(thisParameter.description.get) + val subclass = thisParameter.subclass.map(l => l.head) + val properties = + info + .filter(p => !p.name.startsWith("__")) // remove __this__ + .filter(p => !p.removed.isDefined && (!config.strict || !p.deprecated.isDefined)) // always remove 'removed' arguments and if need be create a strict schema by removing deprecated + .filter(p => !config.strict || !(p.name == "arguments" && (thisParameter.`type` == "Functionality" || thisParameter.`type` == "Config"))) // exception: remove 'arguments' in 'Functionality' for strict schema + val propertiesJson = properties.map(p => { + val pDescription = p.description.map(s => removeMarkup(s)) + val trimmedType = p.`type` match { + case s if s.startsWith("Option[") => s.stripPrefix("Option[").stripSuffix("]") + case s => s + } + + val mapRegex = "(List)?Map\\[String,(\\w*)\\]".r + + implicit val useAllInEither = thisParameter.`type` == "NextflowDirectives" || thisParameter.`type` == "NextflowAuto" + + trimmedType match { + case s"List[$s]" => + (p.name, arrayType(s, pDescription)) + + case "Either[String,List[String]]" => + (p.name, eitherJson( + valueType("String", pDescription), + arrayType("String", pDescription) + )) + + case "Either[Map[String,String],String]" => + (p.name, eitherJson( + mapType("String", pDescription), + valueType("String", pDescription) + )) + + case s"Either[$s,$t]" => + (p.name, eitherJson( + valueType(s, pDescription), + valueType(t, pDescription) + )) + + case "OneOrMore[Map[String,String]]" => + (p.name, oneOrMoreJson( + mapType("String", pDescription) + )) + + case "OneOrMore[Either[String,Map[String,String]]]" => + (p.name, oneOrMoreJson( + eitherJson( + valueType("String", pDescription), + mapType("String", pDescription) + ) + )) + + case s"OneOrMore[$s]" => + if (s == "String" && p.name == "port" && subclass == Some("executable")) { + // Custom exception + // This is the port field for a excutable runner. + // We want to allow a Strings or Ints. + (p.name, eitherJson( + valueType("Int", pDescription), + valueType("String", pDescription), + arrayType("Int", pDescription), + arrayType("String", pDescription) + )) + } else { + (p.name, oneOrMoreType(s, pDescription)) + } + + case mapRegex(_, s) => + (p.name, mapType(s, pDescription)) + + case s if p.name == "type" && subclass.isDefined => + var subclassString = subclass.get.stripSuffix("withname") + if (config.minimal) { + ("type", Json.obj( + "const" -> Json.fromString(subclassString) + )) + } else { + ("type", Json.obj( + "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class + "const" -> Json.fromString(subclassString) + )) + } - // case s => - // (p.name, valueType(s, pDescription)) - // } - - // }) - - // val required = properties.filter(p => - // !( - // p.`type`.startsWith("Option[") || - // p.default.isDefined || - // p.hasUndocumented || - // (p.name == "type" && thisParameter.`type` == "PlainFile" && !config.strict) // Custom exception, file resources are "kind of" default - // ) || - // // Strict schema is what will be outputted by Viash, so fields with a default value other than 'None' will always have a value -> add it in the strict schema as required. - // (!p.`type`.startsWith("Option[") && p.default != Some("Empty") && config.strict && !p.hasUndocumented)) - // val requiredJson = required.map(p => Json.fromString(p.name)) - - // val k = thisParameter.`type` - // val descr = config.minimal match { - // case true => None - // case false => Some(description) - // } - // val v = Json.obj( - // descr.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ - // Seq("type" -> Json.fromString("object"), - // "properties" -> Json.obj(propertiesJson: _*), - // "required" -> Json.arr(requiredJson: _*), - // "additionalProperties" -> Json.False): _* - // ) - // k -> v + case s => + (p.name, valueType(s, pDescription)) + } + + }) + + val required = properties.filter(p => + !( + p.`type`.startsWith("Option[") || + p.default.isDefined || + p.hasUndocumented || + (p.name == "type" && thisParameter.`type` == "PlainFile" && !config.strict) // Custom exception, file resources are "kind of" default + ) || + // Strict schema is what will be outputted by Viash, so fields with a default value other than 'None' will always have a value -> add it in the strict schema as required. + (!p.`type`.startsWith("Option[") && p.default != Some("Empty") && config.strict && !p.hasUndocumented)) + val requiredJson = required.map(p => Json.fromString(p.name)) + + val k = thisParameter.`type` + val descr = config.minimal match { + case true => None + case false => Some(description) + } + val v = Json.obj( + descr.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ + Seq("type" -> Json.fromString("object"), + "properties" -> Json.obj(propertiesJson: _*), + "required" -> Json.arr(requiredJson: _*), + "additionalProperties" -> Json.False): _* + ) + k -> v } def createSuperClassSchema(info: List[ParameterSchema])(implicit config: SchemaConfig): (String, Json) = { From e39672cb7fa78277868d20d96cfb911b5e89dd90 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 13 Sep 2024 08:39:43 +0200 Subject: [PATCH 18/46] fix broken indentation --- .../scala/io/viash/schemas/JsonSchema.scala | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/scala/io/viash/schemas/JsonSchema.scala b/src/main/scala/io/viash/schemas/JsonSchema.scala index eab53af77..ef9d679bc 100644 --- a/src/main/scala/io/viash/schemas/JsonSchema.scala +++ b/src/main/scala/io/viash/schemas/JsonSchema.scala @@ -199,18 +199,18 @@ object JsonSchema { case mapRegex(_, s) => (p.name, mapType(s, pDescription)) - case s if p.name == "type" && subclass.isDefined => - var subclassString = subclass.get.stripSuffix("withname") - if (config.minimal) { - ("type", Json.obj( - "const" -> Json.fromString(subclassString) - )) - } else { - ("type", Json.obj( - "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class - "const" -> Json.fromString(subclassString) - )) - } + case s if p.name == "type" && subclass.isDefined => + var subclassString = subclass.get.stripSuffix("withname") + if (config.minimal) { + ("type", Json.obj( + "const" -> Json.fromString(subclassString) + )) + } else { + ("type", Json.obj( + "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class + "const" -> Json.fromString(subclassString) + )) + } case s => (p.name, valueType(s, pDescription)) From fd79c50a40a5b902b4668ece0a04477d102d20d4 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 27 Sep 2024 19:39:57 +0200 Subject: [PATCH 19/46] Start progressing to Circe to use mirroring directly instead of going through the schemas super-classes --- .../scala/io/viash/helpers/Mirroring.scala | 2 +- ...onfiguredDecoderWithDeprecationCheck.scala | 143 +++++++++--------- ...ConfiguredDecoderWithValidationCheck.scala | 7 +- .../circe/DeriveConfiguredEncoderStrict.scala | 18 +-- .../io/viash/schemas/CollectedSchemas.scala | 28 +++- 5 files changed, 102 insertions(+), 96 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 97a7e9e92..06193137d 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -31,4 +31,4 @@ def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = def niceNameImpl[T: Type](using Quotes): Expr[String] = import quotes.reflect.* val name = TypeRepr.of[T].typeSymbol.name - Expr(name) \ No newline at end of file + Expr(name) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index aac6e99d1..b2ccd19a9 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -17,86 +17,83 @@ package io.viash.helpers.circe -import io.circe.{ Decoder, CursorOp } - +import io.circe.{ ACursor, Decoder, CursorOp } import io.circe.derivation.{Configuration, ConfiguredDecoder} import scala.deriving.Mirror -import io.viash.schemas.ParameterSchema -import io.circe.ACursor - import io.viash.helpers.Logging -import io.viash.schemas.CollectedSchemas object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { - // private def memberDeprecationCheck(name: String, history: List[CursorOp], parameters: List[ParameterSchema]): Unit = { - // val schema = parameters.find(p => p.name == name).getOrElse(ParameterSchema("", "", "", None, None, None, None, None, None, None, None, false, false)) - - // lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - - // lazy val fullHistoryName = - // if (historyString.isEmpty) { - // s".$name" - // } else { - // s".$historyString.$name" - // } - - // schema.deprecated match { - // case Some(d) => - // info(s"Warning: $fullHistoryName is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") - // case _ => - // } - // schema.removed match { - // case Some(r) => - // info(s"Error: $fullHistoryName was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") - // case _ => - // } - // if (schema.hasInternalFunctionality) { - // error(s"Error: $fullHistoryName is internal functionality.") - // throw new RuntimeException(s"Internal functionality used: $fullHistoryName") - // } - // } - - // private def selfDeprecationCheck(parameters: List[ParameterSchema]): Unit = { - // val schema = parameters.find(p => p.name == "__this__").get - - // schema.deprecated match { - // case Some(d) => - // info(s"Warning: ${schema.`type`} is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") - // case _ => - // } - // schema.removed match { - // case Some(r) => - // info(s"Error: ${schema.`type`} was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") - // case _ => - // } - // } - - // // - // def checkDeprecation[A](cursor: ACursor)(implicit tag: TypeTag[A]) : ACursor = { - // val parameters = CollectedSchemas.getParameters[A]() - - // selfDeprecationCheck(parameters) - - // // check each defined 'key' value - // for (key <- cursor.keys.getOrElse(Nil)) { - // val isEmpty = - // cursor.downField(key).focus.get match { - // case value if value.isNull => true - // case value if value.isArray => value.asArray.get.isEmpty - // case value if value.isObject => value.asObject.get.isEmpty - // case _ => false - // } - // if (!isEmpty) { - // memberDeprecationCheck(key, cursor.history, parameters) - // } - // } - // cursor // return unchanged json info - // } - - def checkDeprecation[A](cursor: ACursor): ACursor = { - cursor + private def memberDeprecationCheck[A](name: String, history: List[CursorOp])(using A: Mirror.Of[A]): Unit = { + lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") + + lazy val fullHistoryName = + if (historyString.isEmpty) { + s".$name" + } else { + s".$historyString.$name" + } + + // TODO + val deprecated = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] + val removed = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] + val hasInternalFunctionality = false + + deprecated match { + case Some(d) => + info(s"Warning: $fullHistoryName is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + case _ => + } + removed match { + case Some(r) => + info(s"Error: $fullHistoryName was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + case _ => + } + if (hasInternalFunctionality) { + error(s"Error: $fullHistoryName is internal functionality.") + throw new RuntimeException(s"Internal functionality used: $fullHistoryName") + } + } + + private def selfDeprecationCheck[A]()(using A: Mirror.Of[A]): Unit = { + + // TODO + val name = "" + val deprecated = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] + val removed = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] + + deprecated match { + case Some(d) => + info(s"Warning: $name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + case _ => + } + removed match { + case Some(r) => + info(s"Error: $name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + case _ => + } + + } + + def checkDeprecation[A](cursor: ACursor)(using A: Mirror.Of[A]) : ACursor = { + + selfDeprecationCheck() + + // check each defined 'key' value + for (key <- cursor.keys.getOrElse(Nil)) { + val isEmpty = + cursor.downField(key).focus.get match { + case value if value.isNull => true + case value if value.isArray => value.asArray.get.isEmpty + case value if value.isObject => value.asObject.get.isEmpty + case _ => false + } + if (!isEmpty) { + memberDeprecationCheck(key, cursor.history) + } + } + cursor // return unchanged json info } // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 6e78db2e2..1b939524f 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -17,15 +17,12 @@ package io.viash.helpers.circe -import io.circe.{ Decoder, CursorOp } - +import io.circe.{ Decoder, CursorOp, HCursor } import io.circe.derivation.{Configuration, ConfiguredDecoder} import scala.deriving.Mirror -import io.viash.schemas.ParameterSchema -import io.circe.ACursor + import io.viash.exceptions.ConfigParserSubTypeException import io.viash.exceptions.ConfigParserValidationException -import io.circe.HCursor import io.viash.helpers.{niceNameOf, fieldsOf} object DeriveConfiguredDecoderWithValidationCheck { diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala index 706f63c57..c8cb6502a 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala @@ -21,23 +21,19 @@ import io.circe.Encoder import io.circe.derivation.{Configuration, ConfiguredEncoder} import scala.deriving.Mirror -// import io.circe.{Encoder, Json, HCursor} - -// import scala.reflect.runtime.universe._ -// import shapeless.Lazy -import io.viash.schemas.ParameterSchema -import io.viash.schemas.CollectedSchemas - object DeriveConfiguredEncoderStrict { inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredEncoder[A] - // .mapJsonObject{ jsonObject => + .mapJsonObject{ jsonObject => // val parameters = CollectedSchemas.getParameters[T]() - // jsonObject.filterKeys( k => + jsonObject.filterKeys( k => + // TODO + val hasInternalFunctionality = false + !hasInternalFunctionality // parameters // .find(_.name == k) // find the correct parameter // .map(!_.hasInternalFunctionality) // check if it has the 'internalFunctionality' annotation // .getOrElse(true) // fallback, shouldn't really happen - // ) - // } + ) + } } diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index a97755a81..3c0ed2946 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -42,6 +42,8 @@ import io.viash.config.Author import io.viash.config.ComputationalRequirements import io.viash.config.Links import io.viash.config.References +import scala.deriving.Mirror +import scala.compiletime.{ codeOf, constValue, erasedValue, error, summonFrom, summonInline } final case class CollectedSchemas ( config: Map[String, List[ParameterSchema]], @@ -81,15 +83,20 @@ object CollectedSchemas { private implicit val encodeDeprecatedOrRemoved: Encoder.AsObject[DeprecatedOrRemovedSchema] = deriveConfiguredEncoder private implicit val encodeExample: Encoder.AsObject[ExampleSchema] = deriveConfiguredEncoder - private def getMembers[T/*: TypeTag*/](): (Map[String,List[MemberInfo]], List[Symbol]) = { - (Map.empty, Nil) + private inline final def summonLabels[T <: Tuple]: List[String] = + inline erasedValue[T] match + case _: EmptyTuple => Nil + case _: (t *: ts) => constValue[t].asInstanceOf[String] :: summonLabels[ts] + + private def getMembers[T /*TypeTag*/]/*(using mirror: Mirror.Of[T])*/(): (Map[String,List[MemberInfo]], List[Symbol]) = { - // val name = typeOf[T].typeSymbol.shortName + // val name: String = constValue[mirror.MirroredLabel] - // // Get all members and filter for constructors, first one should be the best (most complete) one - // // Traits don't have constructors - // // Get all parameters and store their short name + // Get all members and filter for constructors, first one should be the best (most complete) one + // Traits don't have constructors + // Get all parameters and store their short name // val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String]) + // val constructorMembers = summonLabels[mirror.MirroredElemLabels] // val baseClasses = typeOf[T].baseClasses // .filter(_.fullName.startsWith("io.viash")) @@ -117,6 +124,15 @@ object CollectedSchemas { // .groupBy(k => k.shortName) // (allMembers, baseClasses) + + // println(s"name: $name") + // println(s"constructorMembers: $constructorMembers") + // println(s"baseClasses: $baseClasses") + // println(s"documentFully: $documentFully") + // println(s"memberNames: $memberNames") + // println(s"allMembers: $allMembers") + + (Map.empty, Nil) } lazy val schemaClasses = List( From 4fde75b5bd7f5bd25b6c5e6165b6d9858a90a4d6 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 27 Sep 2024 21:42:04 +0200 Subject: [PATCH 20/46] Add implementation of fieldsInternalFunctionality Add some minor type annotations in CollectedSchemas, which will refactoring easier because we'll now know what was previously expected --- .../scala/io/viash/helpers/Mirroring.scala | 19 ++++++++++++++++--- .../circe/DeriveConfiguredEncoderStrict.scala | 13 +++---------- .../io/viash/schemas/CollectedSchemas.scala | 6 +++--- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 06193137d..94b6e1ea9 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -18,17 +18,30 @@ package io.viash.helpers import scala.quoted.* +import io.viash.schemas.internalFunctionality inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } inline def niceNameOf[T]: String = ${ niceNameImpl[T] } +inline def fieldsInternalFunctionality[T]: List[String] = ${ fieldsInternalFunctionalityImpl[T] } def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = import quotes.reflect.* - val cls = TypeRepr.of[T].classSymbol.get - val fieldSymbols = cls.caseFields.map(_.name) + val tpe = TypeRepr.of[T].typeSymbol + val fieldSymbols = tpe.caseFields.map(_.name) Expr(fieldSymbols) def niceNameImpl[T: Type](using Quotes): Expr[String] = import quotes.reflect.* val name = TypeRepr.of[T].typeSymbol.name - Expr(name) + Expr(name) + +// TODO this doesn't get annotations from parent classes +def fieldsInternalFunctionalityImpl[T: Type](using Quotes): Expr[List[String]] = + import quotes.reflect.* + val annot = TypeRepr.of[internalFunctionality].typeSymbol + val fieldSymbols = TypeRepr + .of[T] + .baseClasses + .flatMap(_.declaredFields) + .collect{case f if f.hasAnnotation(annot) => f.name } + Expr(fieldSymbols) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala index c8cb6502a..ffe4b41d6 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala @@ -20,20 +20,13 @@ package io.viash.helpers.circe import io.circe.Encoder import io.circe.derivation.{Configuration, ConfiguredEncoder} import scala.deriving.Mirror +import io.viash.helpers.fieldsInternalFunctionality object DeriveConfiguredEncoderStrict { inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredEncoder[A] .mapJsonObject{ jsonObject => - // val parameters = CollectedSchemas.getParameters[T]() - jsonObject.filterKeys( k => - // TODO - val hasInternalFunctionality = false - !hasInternalFunctionality - // parameters - // .find(_.name == k) // find the correct parameter - // .map(!_.hasInternalFunctionality) // check if it has the 'internalFunctionality' annotation - // .getOrElse(true) // fallback, shouldn't really happen - ) + val fieldMap = fieldsInternalFunctionality[A] + jsonObject.filterKeys(k => !fieldMap.contains(k)) } } diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 3c0ed2946..6a6001df0 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -219,7 +219,7 @@ object CollectedSchemas { .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") } - // private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]) = { + // private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]): List[(String, String, List[String], List[Annotation])] = { // val annMembers = members // .map{ case (memberName, memberInfo) => { // val h = memberInfo.head @@ -233,14 +233,14 @@ object CollectedSchemas { // .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class // } - // private val getSchema = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { + // private val getSchema: ((Map[String,List[MemberInfo]], List[Symbol])) => List[ParameterSchema] = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { // case (members, classes) => { // annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } // } // } // get all parameters for a given type, including parent class annotations - // def getParameters[T: TypeTag]() = getSchema(getMembers[T]()) + // def getParameters[T: TypeTag](): List[ParameterSchema] = getSchema(getMembers[T]()) // Main call for documentation output lazy val fullData: List[List[ParameterSchema]] = Nil//schemaClasses.map{ v => getSchema(v)} From f7e2d0bf803df06dd2cf1408b0130a6f43ec66c0 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Sat, 28 Sep 2024 11:07:09 +0200 Subject: [PATCH 21/46] add deprecation & removal annotation code seems likely that the code can be streamlined a bit more, tbd --- .../scala/io/viash/helpers/Mirroring.scala | 86 +++++++++++++++++-- ...onfiguredDecoderWithDeprecationCheck.scala | 41 +++++---- src/main/scala/io/viash/schemas/package.scala | 4 +- 3 files changed, 102 insertions(+), 29 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 94b6e1ea9..5d493f580 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -18,11 +18,57 @@ package io.viash.helpers import scala.quoted.* -import io.viash.schemas.internalFunctionality +import io.viash.schemas.{deprecated, internalFunctionality, removed} -inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } inline def niceNameOf[T]: String = ${ niceNameImpl[T] } +inline def deprecatedOf[T]: Vector[(String, String, String)] = ${ deprecatedOfImpl[T] } +inline def removedOf[T]: Vector[(String, String, String)] = ${ removedOfImpl[T] } + +inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } inline def fieldsInternalFunctionality[T]: List[String] = ${ fieldsInternalFunctionalityImpl[T] } +inline def fieldsDeprecated[T]: Vector[(String, String, String, String)] = ${ fieldsDeprecatedImpl[T] } +inline def fieldsRemoved[T]: Vector[(String, String, String, String)] = ${ fieldsRemovedImpl[T] } + +def niceNameImpl[T: Type](using Quotes): Expr[String] = + import quotes.reflect.* + val name = TypeRepr.of[T].typeSymbol.name + Expr(name) + +def deprecatedOfImpl[T](using Type[T], Quotes): Expr[Vector[(String, String, String)]] = + import quotes.reflect.* + val annot = TypeRepr.of[deprecated].typeSymbol + val tuple = TypeRepr + .of[T] + .typeSymbol + .getAnnotation(annot) + .map: + case annot => + val annotExpr = annot.asExprOf[deprecated] + '{ ($annotExpr.message, $annotExpr.since, $annotExpr.plannedRemoval) } + val list = tuple match { + case Some(t) => Seq(t) + case None => Nil + } + val seq: Expr[Seq[(String, String, String)]] = Expr.ofSeq(list) + '{ $seq.toVector } + +def removedOfImpl[T](using Type[T], Quotes): Expr[Vector[(String, String, String)]] = + import quotes.reflect.* + val annot = TypeRepr.of[removed].typeSymbol + val tuple = TypeRepr + .of[T] + .typeSymbol + .getAnnotation(annot) + .map: + case annot => + val annotExpr = annot.asExprOf[removed] + '{ ($annotExpr.message, $annotExpr.deprecatedSince, $annotExpr.since) } + val list = tuple match { + case Some(t) => Seq(t) + case None => Nil + } + val seq: Expr[Seq[(String, String, String)]] = Expr.ofSeq(list) + '{ $seq.toVector } def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = import quotes.reflect.* @@ -30,12 +76,6 @@ def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = val fieldSymbols = tpe.caseFields.map(_.name) Expr(fieldSymbols) -def niceNameImpl[T: Type](using Quotes): Expr[String] = - import quotes.reflect.* - val name = TypeRepr.of[T].typeSymbol.name - Expr(name) - -// TODO this doesn't get annotations from parent classes def fieldsInternalFunctionalityImpl[T: Type](using Quotes): Expr[List[String]] = import quotes.reflect.* val annot = TypeRepr.of[internalFunctionality].typeSymbol @@ -45,3 +85,33 @@ def fieldsInternalFunctionalityImpl[T: Type](using Quotes): Expr[List[String]] = .flatMap(_.declaredFields) .collect{case f if f.hasAnnotation(annot) => f.name } Expr(fieldSymbols) + +def fieldsDeprecatedImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] = + import quotes.reflect.* + val annot = TypeRepr.of[deprecated].typeSymbol + val tuples = TypeRepr + .of[T] + .baseClasses + .flatMap(_.declaredFields) + .collect: + case f if f.hasAnnotation(annot) => + val fieldNameExpr = Expr(f.name.asInstanceOf[String]) + val annotExpr = f.getAnnotation(annot).get.asExprOf[deprecated] + '{ ($fieldNameExpr, $annotExpr.message, $annotExpr.since, $annotExpr.plannedRemoval) } + val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples) + '{ $seq.toVector } + +def fieldsRemovedImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] = + import quotes.reflect.* + val annot = TypeRepr.of[removed].typeSymbol + val tuples = TypeRepr + .of[T] + .baseClasses + .flatMap(_.declaredFields) + .collect: + case f if f.hasAnnotation(annot) => + val fieldNameExpr = Expr(f.name.asInstanceOf[String]) + val annotExpr = f.getAnnotation(annot).get.asExprOf[removed] + '{ ($fieldNameExpr, $annotExpr.message, $annotExpr.deprecatedSince, $annotExpr.since) } + val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples) + '{ $seq.toVector } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index b2ccd19a9..2efbde452 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -22,10 +22,17 @@ import io.circe.derivation.{Configuration, ConfiguredDecoder} import scala.deriving.Mirror import io.viash.helpers.Logging +import io.viash.helpers.* object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { - private def memberDeprecationCheck[A](name: String, history: List[CursorOp])(using A: Mirror.Of[A]): Unit = { + private inline def memberDeprecationCheck[A]( + name: String, + history: List[CursorOp], + deprecated: Option[(String, String, String)], + removed: Option[(String, String, String)], + hasInternalFunctionality: Boolean + )(using inline A: Mirror.Of[A]): Unit = { lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") lazy val fullHistoryName = @@ -35,19 +42,14 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { s".$historyString.$name" } - // TODO - val deprecated = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] - val removed = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] - val hasInternalFunctionality = false - deprecated match { case Some(d) => - info(s"Warning: $fullHistoryName is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + info(s"Warning: $fullHistoryName is deprecated: ${d._1} Deprecated since ${d._2}, planned removal ${d._3}.") case _ => } removed match { case Some(r) => - info(s"Error: $fullHistoryName was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + info(s"Error: $fullHistoryName was removed: ${r._1} Initially deprecated ${r._2}, removed ${r._3}.") case _ => } if (hasInternalFunctionality) { @@ -56,30 +58,31 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { } } - private def selfDeprecationCheck[A]()(using A: Mirror.Of[A]): Unit = { - - // TODO - val name = "" - val deprecated = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] - val removed = Option.empty[io.viash.schemas.DeprecatedOrRemovedSchema] + private inline def selfDeprecationCheck[A]()(using inline A: Mirror.Of[A]): Unit = { + val name = niceNameOf[A] + val deprecated = deprecatedOf[A].headOption + val removed = removedOf[A].headOption deprecated match { case Some(d) => - info(s"Warning: $name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + info(s"Warning: $name is deprecated: ${d._1} Deprecated since ${d._2}, planned removal ${d._3}.") case _ => } removed match { case Some(r) => - info(s"Error: $name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + info(s"Error: $name was removed: ${r._1} Initially deprecated ${r._2}, removed ${r._3}.") case _ => } - } - def checkDeprecation[A](cursor: ACursor)(using A: Mirror.Of[A]) : ACursor = { + inline def checkDeprecation[A](cursor: ACursor)(using inline A: Mirror.Of[A]) : ACursor = { selfDeprecationCheck() + val fd = fieldsDeprecated[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap + val fr = fieldsRemoved[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap + val fif = fieldsInternalFunctionality[A] + // check each defined 'key' value for (key <- cursor.keys.getOrElse(Nil)) { val isEmpty = @@ -90,7 +93,7 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { case _ => false } if (!isEmpty) { - memberDeprecationCheck(key, cursor.history) + memberDeprecationCheck(key, cursor.history, fd.get(key), fr.get(key), fif.contains(key)) } } cursor // return unchanged json info diff --git a/src/main/scala/io/viash/schemas/package.scala b/src/main/scala/io/viash/schemas/package.scala index eb6210f5f..6011e3198 100644 --- a/src/main/scala/io/viash/schemas/package.scala +++ b/src/main/scala/io/viash/schemas/package.scala @@ -33,10 +33,10 @@ package object schemas { class description(example: String) extends scala.annotation.StaticAnnotation @getter @setter @beanGetter @beanSetter @field - class deprecated(message: String, since: String, plannedRemoval: String) extends scala.annotation.StaticAnnotation + class deprecated(val message: String, val since: String, val plannedRemoval: String) extends scala.annotation.StaticAnnotation @getter @setter @beanGetter @beanSetter @field - class removed(message: String, deprecatedSince: String, since: String) extends scala.annotation.StaticAnnotation + class removed(val message: String, val deprecatedSince: String, val since: String) extends scala.annotation.StaticAnnotation @getter @setter @beanGetter @beanSetter @field class default(default: String) extends scala.annotation.StaticAnnotation From ba38803014f49ef3912ebaa7bc0ad2f7b6a9ff58 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Sat, 28 Sep 2024 13:13:57 +0200 Subject: [PATCH 22/46] Refactor mirror method names --- src/main/scala/io/viash/helpers/Mirroring.scala | 16 ++++++++-------- ...veConfiguredDecoderWithDeprecationCheck.scala | 8 ++++---- .../circe/DeriveConfiguredEncoderStrict.scala | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 5d493f580..20d7b0a7f 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -20,16 +20,16 @@ package io.viash.helpers import scala.quoted.* import io.viash.schemas.{deprecated, internalFunctionality, removed} -inline def niceNameOf[T]: String = ${ niceNameImpl[T] } +inline def niceNameOf[T]: String = ${ niceNameOfImpl[T] } inline def deprecatedOf[T]: Vector[(String, String, String)] = ${ deprecatedOfImpl[T] } inline def removedOf[T]: Vector[(String, String, String)] = ${ removedOfImpl[T] } inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } -inline def fieldsInternalFunctionality[T]: List[String] = ${ fieldsInternalFunctionalityImpl[T] } -inline def fieldsDeprecated[T]: Vector[(String, String, String, String)] = ${ fieldsDeprecatedImpl[T] } -inline def fieldsRemoved[T]: Vector[(String, String, String, String)] = ${ fieldsRemovedImpl[T] } +inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctionalityFieldsOfImpl[T] } +inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] } +inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } -def niceNameImpl[T: Type](using Quotes): Expr[String] = +def niceNameOfImpl[T: Type](using Quotes): Expr[String] = import quotes.reflect.* val name = TypeRepr.of[T].typeSymbol.name Expr(name) @@ -76,7 +76,7 @@ def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = val fieldSymbols = tpe.caseFields.map(_.name) Expr(fieldSymbols) -def fieldsInternalFunctionalityImpl[T: Type](using Quotes): Expr[List[String]] = +def internalFunctionalityFieldsOfImpl[T: Type](using Quotes): Expr[List[String]] = import quotes.reflect.* val annot = TypeRepr.of[internalFunctionality].typeSymbol val fieldSymbols = TypeRepr @@ -86,7 +86,7 @@ def fieldsInternalFunctionalityImpl[T: Type](using Quotes): Expr[List[String]] = .collect{case f if f.hasAnnotation(annot) => f.name } Expr(fieldSymbols) -def fieldsDeprecatedImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] = +def deprecatedFieldsOfImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] = import quotes.reflect.* val annot = TypeRepr.of[deprecated].typeSymbol val tuples = TypeRepr @@ -101,7 +101,7 @@ def fieldsDeprecatedImpl[T: Type](using Quotes): Expr[Vector[(String, String, St val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples) '{ $seq.toVector } -def fieldsRemovedImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] = +def removedFieldsOfImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] = import quotes.reflect.* val annot = TypeRepr.of[removed].typeSymbol val tuples = TypeRepr diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 2efbde452..e3978fb3c 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -79,9 +79,9 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { selfDeprecationCheck() - val fd = fieldsDeprecated[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap - val fr = fieldsRemoved[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap - val fif = fieldsInternalFunctionality[A] + val df = deprecatedFieldsOf[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap + val rf = removedFieldsOf[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap + val iff = internalFunctionalityFieldsOf[A] // check each defined 'key' value for (key <- cursor.keys.getOrElse(Nil)) { @@ -93,7 +93,7 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { case _ => false } if (!isEmpty) { - memberDeprecationCheck(key, cursor.history, fd.get(key), fr.get(key), fif.contains(key)) + memberDeprecationCheck(key, cursor.history, df.get(key), rf.get(key), iff.contains(key)) } } cursor // return unchanged json info diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala index ffe4b41d6..fcc595aaa 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala @@ -20,13 +20,13 @@ package io.viash.helpers.circe import io.circe.Encoder import io.circe.derivation.{Configuration, ConfiguredEncoder} import scala.deriving.Mirror -import io.viash.helpers.fieldsInternalFunctionality +import io.viash.helpers.internalFunctionalityFieldsOf object DeriveConfiguredEncoderStrict { inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredEncoder[A] .mapJsonObject{ jsonObject => - val fieldMap = fieldsInternalFunctionality[A] + val fieldMap = internalFunctionalityFieldsOf[A] jsonObject.filterKeys(k => !fieldMap.contains(k)) } } From 0d986395a88a7fae2e945b2b9895666214eaf92d Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Sat, 28 Sep 2024 19:54:36 +0200 Subject: [PATCH 23/46] Fix empty package configs failing as it gets parsed as `false` --- src/main/scala/io/viash/packageConfig/PackageConfig.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/scala/io/viash/packageConfig/PackageConfig.scala b/src/main/scala/io/viash/packageConfig/PackageConfig.scala index 0178eee98..e47fb86c6 100644 --- a/src/main/scala/io/viash/packageConfig/PackageConfig.scala +++ b/src/main/scala/io/viash/packageConfig/PackageConfig.scala @@ -246,7 +246,12 @@ object PackageConfig { /* PACKAGE 0: converted from json */ // convert Json into ViashPackage - val pack0 = Convert.jsonToClass[PackageConfig](json, path.toString()) + // val pack0 = Convert.jsonToClass[PackageConfig](json2, path.toString()) + // TODO fix empty json getting parsed as 'false' and then failing to create a PackageConfig from that + val pack0 = json match { + case json if json == Json.False => PackageConfig() + case json => Convert.jsonToClass[PackageConfig](json, path.toString()) + } /* PACKAGE 1: make resources absolute */ // make paths absolute From 7d84d7dea224476fb8bc6145bf61219eda6d9d13 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Sat, 28 Sep 2024 20:59:43 +0200 Subject: [PATCH 24/46] rename `niceNameOf` to `typeOf` and improve output of e.g. List[String] --- .../io/viash/cli/DocumentedSubcommand.scala | 18 ++++++++---------- .../scala/io/viash/helpers/Mirroring.scala | 18 ++++++++++++++---- .../DeriveConfiguredDecoderFullChecks.scala | 4 ++-- ...ConfiguredDecoderWithDeprecationCheck.scala | 2 +- ...eConfiguredDecoderWithValidationCheck.scala | 6 +++--- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/main/scala/io/viash/cli/DocumentedSubcommand.scala b/src/main/scala/io/viash/cli/DocumentedSubcommand.scala index a1439678a..4ba9b2369 100644 --- a/src/main/scala/io/viash/cli/DocumentedSubcommand.scala +++ b/src/main/scala/io/viash/cli/DocumentedSubcommand.scala @@ -21,7 +21,7 @@ import org.rogach.scallop.Subcommand import org.rogach.scallop.ScallopOptionGroup import org.rogach.scallop.ValueConverter import org.rogach.scallop.ScallopOption -// import scala.reflect.runtime.universe._ +import io.viash.helpers.typeOf /** * Wrapper class for Subcommand to expose protected members @@ -76,7 +76,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co // We need to get the TypeTag[A], which changes the interface of 'opt', however since we have default values we can't just overload the methods. // The same goes for 'trailArgs'. Not really for 'choice' but it's better to keep the same change in naming schema here too. - def registerOpt[A]( + inline def registerOpt[A]( name: String, short: Option[Char] = None, descr: String = "", @@ -86,9 +86,8 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co argName: String = "arg", hidden: Boolean = false, group: ScallopOptionGroup = null - )(implicit conv:ValueConverter[A]/*, tag: TypeTag[A]*/): ScallopOption[A] = { + )(implicit conv:ValueConverter[A]): ScallopOption[A] = { - // val `type` = tag.tpe val cleanName = name match { case null => "" case _ => name @@ -103,14 +102,14 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co argName = Some(argName), hidden = hidden, choices = None, - `type` = "TODO REWORK REFLECTION",//`type`.toString(), + `type` = typeOf[A], optType = "opt" ) registeredOpts = registeredOpts :+ registeredOpt opt(name, short.getOrElse('\u0000'), removeMarkup(descr), default, validate, required, argName, hidden, short.isEmpty, group) } - def registerChoice( + inline def registerChoice( choices: Seq[String], name: String, short: Option[Char], @@ -143,7 +142,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co choice(choices, name, short.getOrElse('\u0000'), removeMarkup(descr), default, required, argName, hidden, short.isEmpty, group) } - def registerTrailArg[A]( + inline def registerTrailArg[A]( name: String, descr: String = "", validate: A => Boolean = (_:A) => true, @@ -151,9 +150,8 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co default: => Option[A] = None, hidden: Boolean = false, group: ScallopOptionGroup = null - )(implicit conv:ValueConverter[A]/*, tag: TypeTag[A]*/) = { + )(implicit conv:ValueConverter[A]) = { - // val `type` = tag.tpe val cleanName = name match { case null => "" case _ => name @@ -168,7 +166,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co argName = None, hidden = hidden, choices = None, - `type` = "TODO REWORK REFLECTION",//`type`.toString, + `type` = typeOf[A], optType = "trailArgs" ) registeredOpts = registeredOpts :+ registeredOpt diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 20d7b0a7f..7131224e0 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -20,7 +20,7 @@ package io.viash.helpers import scala.quoted.* import io.viash.schemas.{deprecated, internalFunctionality, removed} -inline def niceNameOf[T]: String = ${ niceNameOfImpl[T] } +inline def typeOf[T]: String = ${ typeOfImpl[T] } inline def deprecatedOf[T]: Vector[(String, String, String)] = ${ deprecatedOfImpl[T] } inline def removedOf[T]: Vector[(String, String, String)] = ${ removedOfImpl[T] } @@ -29,10 +29,20 @@ inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctiona inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] } inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } -def niceNameOfImpl[T: Type](using Quotes): Expr[String] = +def typeOfImpl[T: Type](using Quotes): Expr[String] = import quotes.reflect.* - val name = TypeRepr.of[T].typeSymbol.name - Expr(name) + val typeRepr = TypeRepr.of[T] + + // Use pattern matching to extract a simplified name + def simpleName(tpe: TypeRepr): String = tpe match { + case AppliedType(tycon, args) => + // If it's a type constructor with arguments, show it in a readable form + s"${simpleName(tycon)}[${args.map(simpleName).mkString(", ")}]" + case _ => + // Strip the full package name to get the simple type name + tpe.typeSymbol.name + } + Expr(simpleName(typeRepr)) def deprecatedOfImpl[T](using Type[T], Quotes): Expr[Vector[(String, String, String)]] = import quotes.reflect.* diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala index 20149f631..6913f6999 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -20,7 +20,7 @@ package io.viash.helpers.circe import io.circe.Decoder import io.circe.derivation.Configuration import scala.deriving.Mirror -import io.viash.helpers.niceNameOf +import io.viash.helpers.typeOf object DeriveConfiguredDecoderFullChecks { import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation @@ -29,7 +29,7 @@ object DeriveConfiguredDecoderFullChecks { inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A] .validate( validator[A], - s"Could not convert json to ${niceNameOf[A]}." + s"Could not convert json to ${typeOf[A]}." ) .prepare( checkDeprecation[A] ) } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index e3978fb3c..d117d4e0c 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -59,7 +59,7 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { } private inline def selfDeprecationCheck[A]()(using inline A: Mirror.Of[A]): Unit = { - val name = niceNameOf[A] + val name = typeOf[A] val deprecated = deprecatedOf[A].headOption val removed = removedOf[A].headOption diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 1b939524f..0780c75f0 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -23,7 +23,7 @@ import scala.deriving.Mirror import io.viash.exceptions.ConfigParserSubTypeException import io.viash.exceptions.ConfigParserValidationException -import io.viash.helpers.{niceNameOf, fieldsOf} +import io.viash.helpers.{typeOf, fieldsOf} object DeriveConfiguredDecoderWithValidationCheck { @@ -59,7 +59,7 @@ object DeriveConfiguredDecoderWithValidationCheck { case _ => None } - throw new ConfigParserValidationException(niceNameOf[A], pred.value.toString(), hint) + throw new ConfigParserValidationException(typeOf[A], pred.value.toString(), hint) false }, _ => true) } @@ -68,7 +68,7 @@ object DeriveConfiguredDecoderWithValidationCheck { inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] .validate( validator[A], - s"Could not convert json to ${niceNameOf[A]}." + s"Could not convert json to ${typeOf[A]}." ) // Dummy decoder to generate exceptions when an invalid type is specified From df4cd609e95ee4c824fb01155eea89b12d6fc8d9 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 24 Oct 2024 11:54:55 +0200 Subject: [PATCH 25/46] Reduce disabled code by adding more placeholders --- build.sbt | 2 +- src/main/scala/io/viash/config/package.scala | 2 +- .../scala/io/viash/helpers/Mirroring.scala | 8 + .../io/viash/schemas/CollectedSchemas.scala | 23 +-- .../io/viash/schemas/ParameterSchema.scala | 148 +++++++++--------- 5 files changed, 100 insertions(+), 83 deletions(-) diff --git a/build.sbt b/build.sbt index aca98f76a..9564433c7 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ libraryDependencies ++= Seq( "dev.optics" %% "monocle-macro" % "3.1.0" ) -val circeVersion = "0.14.9" +val circeVersion = "0.14.10" libraryDependencies ++= Seq( "io.circe" %% "circe-core", diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index e89736506..5c3c4302d 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -272,7 +272,7 @@ package object config { "Could not convert json to Config." ) - implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder + implicit val encodeBuildInfo: Encoder.AsObject[BuildInfo] = deriveConfiguredEncoder implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks // encoder and decoder for Author diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 7131224e0..fdd0d68ee 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -28,6 +28,7 @@ inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctionalityFieldsOfImpl[T] } inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] } inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } +inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } def typeOfImpl[T: Type](using Quotes): Expr[String] = import quotes.reflect.* @@ -125,3 +126,10 @@ def removedFieldsOfImpl[T: Type](using Quotes): Expr[Vector[(String, String, Str '{ ($fieldNameExpr, $annotExpr.message, $annotExpr.deprecatedSince, $annotExpr.since) } val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples) '{ $seq.toVector } + +def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { + import quotes.reflect.* + val tpe = TypeRepr.of[T].typeSymbol + val memberSymbols = tpe.fieldMembers.map(_.name) + Expr(memberSymbols) +} diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 6a6001df0..3f632d984 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -44,6 +44,7 @@ import io.viash.config.Links import io.viash.config.References import scala.deriving.Mirror import scala.compiletime.{ codeOf, constValue, erasedValue, error, summonFrom, summonInline } +import scala.annotation.Annotation final case class CollectedSchemas ( config: Map[String, List[ParameterSchema]], @@ -88,15 +89,17 @@ object CollectedSchemas { case _: EmptyTuple => Nil case _: (t *: ts) => constValue[t].asInstanceOf[String] :: summonLabels[ts] - private def getMembers[T /*TypeTag*/]/*(using mirror: Mirror.Of[T])*/(): (Map[String,List[MemberInfo]], List[Symbol]) = { + private inline def getMembers[T /*TypeTag*/]/*(using mirror: Mirror.Of[T])*/(): (Map[String,List[MemberInfo]], List[Symbol]) = { // val name: String = constValue[mirror.MirroredLabel] + val name: String = typeOf[T] // Get all members and filter for constructors, first one should be the best (most complete) one // Traits don't have constructors // Get all parameters and store their short name // val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String]) // val constructorMembers = summonLabels[mirror.MirroredElemLabels] + val constructorMembers = fieldsOf[T] // val baseClasses = typeOf[T].baseClasses // .filter(_.fullName.startsWith("io.viash")) @@ -111,6 +114,7 @@ object CollectedSchemas { // .filter(!_.isMethod || documentFully) // .map(_.shortName) // .toSeq + val memberNames = membersOf[T] // val allMembers = baseClasses // .zipWithIndex @@ -219,7 +223,7 @@ object CollectedSchemas { .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") } - // private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]): List[(String, String, List[String], List[Annotation])] = { + private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]): List[(String, String, List[String], List[Annotation])] = { // val annMembers = members // .map{ case (memberName, memberInfo) => { // val h = memberInfo.head @@ -231,19 +235,20 @@ object CollectedSchemas { // val allAnnotations = annThis :: annMembers.toList // allAnnotations // .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class - // } + Nil + } - // private val getSchema: ((Map[String,List[MemberInfo]], List[Symbol])) => List[ParameterSchema] = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { - // case (members, classes) => { - // annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } - // } - // } + private val getSchema: ((Map[String,List[MemberInfo]], List[Symbol])) => List[ParameterSchema] = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { + case (members, classes) => { + annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } + } + } // get all parameters for a given type, including parent class annotations // def getParameters[T: TypeTag](): List[ParameterSchema] = getSchema(getMembers[T]()) // Main call for documentation output - lazy val fullData: List[List[ParameterSchema]] = Nil//schemaClasses.map{ v => getSchema(v)} + lazy val fullData: List[List[ParameterSchema]] = schemaClasses.map{ v => getSchema(v)} lazy val data: List[List[ParameterSchema]] = fullData.map(_.filter(p => !p.hasUndocumented && !p.hasInternalFunctionality)) def getKeyFromParamList(data: List[ParameterSchema]): String = data.find(p => p.name == "__this__").get.`type` diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index f0585d43b..1804ec1ba 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -19,6 +19,7 @@ package io.viash.schemas // import scala.reflect.runtime.universe._ import io.viash.schemas.internalFunctionality +import scala.annotation.Annotation final case class ParameterSchema( name: String, @@ -80,89 +81,92 @@ object ParameterSchema { // (name, values) // } - // def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): ParameterSchema = { + def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): ParameterSchema = { + println(s"ParameterSchema: $name, ${`type`}, $hierarchy, $annotations") - // def beautifyTypeName(s: String): String = { + def beautifyTypeName(s: String): String = { - // // "tpe[a]" -> "(\w*)\[(\w*)\]" - // def regexify(s: String) = s.replace("[", "\\[").replace("]", "\\]").replaceAll("\\w+", "(\\\\w+)").r + // "tpe[a]" -> "(\w*)\[(\w*)\]" + def regexify(s: String) = s.replace("[", "\\[").replace("]", "\\]").replaceAll("\\w+", "(\\\\w+)").r - // val regex0 = regexify("tpe") - // val regex1 = regexify("tpe[a]") - // val regex2 = regexify("tpe[a,b]") - // val regexNested1 = regexify("tpe[a[b,c]]") - // val regexNested2 = regexify("tpe[a[b,c[d,e]]]") - // val regexNested3 = regexify("tpe[a,b[c,d]]") - // val regexNested4 = regexify("tpe[a[b[c,d],e]]") - // val regexNested5 = regexify("tpe[a[b,c[d]]]") - // val regexNested6 = regexify("tpe[a[b,c],d]") - // val regexNested7 = regexify("tpe[a,b[c]]") + val regex0 = regexify("tpe") + val regex1 = regexify("tpe[a]") + val regex2 = regexify("tpe[a,b]") + val regexNested1 = regexify("tpe[a[b,c]]") + val regexNested2 = regexify("tpe[a[b,c[d,e]]]") + val regexNested3 = regexify("tpe[a,b[c,d]]") + val regexNested4 = regexify("tpe[a[b[c,d],e]]") + val regexNested5 = regexify("tpe[a[b,c[d]]]") + val regexNested6 = regexify("tpe[a[b,c],d]") + val regexNested7 = regexify("tpe[a,b[c]]") - // def map(a: String, b: String): String = s"Map of $a to $b" - // def either(a: String, b: String): String = s"""Either $a or $b""" + def map(a: String, b: String): String = s"Map of $a to $b" + def either(a: String, b: String): String = s"""Either $a or $b""" - // s match { - // case regex0(tpe) => s"$tpe" - // case regex1(tpe, subtpe) => s"$tpe of $subtpe" - // case regex2("Map", subtpe1, subtpe2) => map(subtpe1,subtpe2) - // case regex2("ListMap", subtpe1, subtpe2) => map(subtpe1, subtpe2) - // case regex2("Either", subtpe1, subtpe2) => either(subtpe1, subtpe2) - // case regexNested1(tpe, a, b, c) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c]")}" - // case regexNested2(tpe, a, b ,c ,d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d,$e]]")}" - // case regexNested3("Either", a, b, c, d) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c,$d]")) - // case regexNested4(tpe, a, b, c, d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b[$c,$d],$e]")}" - // case regexNested5(tpe, a, b, c, d) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d]]")}" - // case regexNested6("Either", a, b, c, d) => either(beautifyTypeName(s"$a[$b,$c]"), beautifyTypeName(d)) - // case regexNested7("Either", a, b, c) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c]")) - // case _ => s - // } - // } + s match { + case regex0(tpe) => s"$tpe" + case regex1(tpe, subtpe) => s"$tpe of $subtpe" + case regex2("Map", subtpe1, subtpe2) => map(subtpe1,subtpe2) + case regex2("ListMap", subtpe1, subtpe2) => map(subtpe1, subtpe2) + case regex2("Either", subtpe1, subtpe2) => either(subtpe1, subtpe2) + case regexNested1(tpe, a, b, c) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c]")}" + case regexNested2(tpe, a, b ,c ,d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d,$e]]")}" + case regexNested3("Either", a, b, c, d) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c,$d]")) + case regexNested4(tpe, a, b, c, d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b[$c,$d],$e]")}" + case regexNested5(tpe, a, b, c, d) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d]]")}" + case regexNested6("Either", a, b, c, d) => either(beautifyTypeName(s"$a[$b,$c]"), beautifyTypeName(d)) + case regexNested7("Either", a, b, c) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c]")) + case _ => s + } + } - // val annStrings = annotations.map(annotationToStrings(_)) - // val hierarchyOption = hierarchy match { - // case l if l.length > 0 => Some(l) - // case _ => None - // } + // val annStrings = annotations.map(annotationToStrings(_)) + val annStrings = List[(String, List[String])]() // TODO + val hierarchyOption = hierarchy match { + case l if l.length > 0 => Some(l) + case _ => None + } - // // name is e.g. "io.viash.config.Config.name", only keep "name" - // // name can also be "__this__" - // // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. - // val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) - // val nameFromClass = name.split('.').last - // val name_ = (nameOverride, nameFromClass) match { - // case (Some(_), "__this__") => "__this__" - // case (Some(ann), _) => ann - // case (None, name) => name - // } + // name is e.g. "io.viash.config.Config.name", only keep "name" + // name can also be "__this__" + // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. + // val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) + val nameOverride = Option.empty // TODO + val nameFromClass = name.split('.').last + val name_ = (nameOverride, nameFromClass) match { + case (Some(_), "__this__") => "__this__" + case (Some(ann), _) => ann + case (None, name) => name + } - // val typeName = (`type`, nameOverride, nameFromClass) match { - // case (_, Some(newTypeName), "__this__") => newTypeName - // case (typeName, _, _) => typeName - // } + val typeName = (`type`, nameOverride, nameFromClass) match { + case (_, Some(newTypeName), "__this__") => newTypeName + case (typeName, _, _) => typeName + } - // val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head}) - // val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) - // val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) - // val examples = example ::: exampleWithDescription match { - // case l if l.length > 0 => Some(l) - // case _ => None - // } - // val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head}) - // val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) - // val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) - // val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head}) - // val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") - // val default = defaultFromAnnotation orElse defaultFromType - // val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { - // case l if l.nonEmpty => Some(l) - // case _ => None - // } + val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head}) + val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) + val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) + val examples = example ::: exampleWithDescription match { + case l if l.length > 0 => Some(l) + case _ => None + } + val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head}) + val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) + val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) + val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head}) + val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") + val default = defaultFromAnnotation orElse defaultFromType + val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { + case l if l.nonEmpty => Some(l) + case _ => None + } - // val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")} - // val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")} + val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")} + val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")} - // ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass, undocumented, internalFunctionality) - // } + ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass, undocumented, internalFunctionality) + } } final case class DeprecatedOrRemovedSchema( From 737465a31e98e1b005469ff76de15a2097af058b Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 24 Oct 2024 16:51:43 +0200 Subject: [PATCH 26/46] Bump scala to 3.3.4, solve some new warnings and reduce inline code sizes --- build.sbt | 2 +- ...onfiguredDecoderWithDeprecationCheck.scala | 4 +- ...ConfiguredDecoderWithValidationCheck.scala | 50 ++++++++++--------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/build.sbt b/build.sbt index 9564433c7..58c1f930e 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "viash" version := "0.9.0-dev" -scalaVersion := "3.3.3" +scalaVersion := "3.3.4" libraryDependencies ++= Seq( "org.scalactic" %% "scalactic" % "3.2.15" % "test", diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index d117d4e0c..2e0857207 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -26,13 +26,13 @@ import io.viash.helpers.* object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { - private inline def memberDeprecationCheck[A]( + private def memberDeprecationCheck( name: String, history: List[CursorOp], deprecated: Option[(String, String, String)], removed: Option[(String, String, String)], hasInternalFunctionality: Boolean - )(using inline A: Mirror.Of[A]): Unit = { + ): Unit = { lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") lazy val fullHistoryName = diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 0780c75f0..f6ddeafa8 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -17,7 +17,7 @@ package io.viash.helpers.circe -import io.circe.{ Decoder, CursorOp, HCursor } +import io.circe.{ Decoder, CursorOp, HCursor, DecodingFailure } import io.circe.derivation.{Configuration, ConfiguredDecoder} import scala.deriving.Mirror @@ -27,6 +27,28 @@ import io.viash.helpers.{typeOf, fieldsOf} object DeriveConfiguredDecoderWithValidationCheck { + private def validator_static_error(pred: HCursor, validFields: List[String], typeOf: String)(error: DecodingFailure): Boolean = { + val usedFields = pred.value.asObject.map(_.keys.toSeq) + val invalidFields = usedFields.map(_.diff(validFields)) + + val fieldsHint = invalidFields match { + case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}") + case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}") + case _ => None + } + + val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") + + val hint = (fieldsHint, historyString, error.message) match { + case (Some(a), h, _) if h != "" => Some(s".$h -> $a") + case (Some(a), _, _) => Some(a) + case (None, h, m) if h != "" => Some(s".$h -> $m") + case _ => None + } + + throw new ConfigParserValidationException(typeOf, pred.value.toString(), hint) + } + // Validate the json can correctly converted to the required type by actually converting it. // Throw an exception when the conversion fails. inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A], inline configuration: Configuration): Boolean = { @@ -39,29 +61,9 @@ object DeriveConfiguredDecoderWithValidationCheck { case _ => d(pred) } - v.fold(error => { - val usedFields = pred.value.asObject.map(_.keys.toSeq) - val validFields = fieldsOf[A] - val invalidFields = usedFields.map(_.diff(validFields)) - - val fieldsHint = invalidFields match { - case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}") - case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}") - case _ => None - } - - val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - - val hint = (fieldsHint, historyString, error.message) match { - case (Some(a), h, _) if h != "" => Some(s".$h -> $a") - case (Some(a), _, _) => Some(a) - case (None, h, m) if h != "" => Some(s".$h -> $m") - case _ => None - } - - throw new ConfigParserValidationException(typeOf[A], pred.value.toString(), hint) - false - }, _ => true) + v.fold( + validator_static_error(pred, fieldsOf[A], typeOf[A]), + _ => true) } // Attempts to convert the json to the desired class. Throw an exception if the conversion fails. From b5bdee48e8240f82c2dbb977b2e80dc5aea8fb2c Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 25 Oct 2024 09:25:57 +0200 Subject: [PATCH 27/46] Add code to get class annotations, step towards getting it for members too --- .../scala/io/viash/helpers/Mirroring.scala | 60 ++++++++++++++++++- .../io/viash/schemas/ParameterSchema.scala | 3 +- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index fdd0d68ee..e1a9b7d6f 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -18,7 +18,7 @@ package io.viash.helpers import scala.quoted.* -import io.viash.schemas.{deprecated, internalFunctionality, removed} +import io.viash.schemas.* inline def typeOf[T]: String = ${ typeOfImpl[T] } inline def deprecatedOf[T]: Vector[(String, String, String)] = ${ deprecatedOfImpl[T] } @@ -28,6 +28,7 @@ inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] } inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctionalityFieldsOfImpl[T] } inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] } inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } +inline def annotationsOf[T]: List[(String, List[String])] = ${ annotationsOfImpl[T] } inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } def typeOfImpl[T: Type](using Quotes): Expr[String] = @@ -127,6 +128,63 @@ def removedFieldsOfImpl[T: Type](using Quotes): Expr[Vector[(String, String, Str val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples) '{ $seq.toVector } + + +def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] = + import quotes.reflect.* + val tpe = TypeRepr.of[T].typeSymbol + + + def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = { + s.replaceAll("\\\\n", "\n").stripMargin(marginChar) + } + + def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { + l.map(i => i match { + // case Literal(Constant(value: String)) => + // unfinishedStringStripMargin(value, marginChar) + case Literal(value) => + unfinishedStringStripMargin(value.show(using Printer.ConstantCode), marginChar) + case _ => + "unmatched in mapTreeList: " + i.toString() + }).mkString + } + + // Traverse tree information and extract values or lists of values + def annotationToStrings(ann: Term): List[String] = { + // val name = ann.tree.tpe.toString() + val values = ann match { + case Apply(c, args: List[Tree]) => + args.collect({ + case i: Tree => + i match { + // Here 'Apply' contains lists + // While 'Select' has a single element + // case Literal(Constant(value: String)) => + // value + case Literal(value) => + value.show(using Printer.ConstantCode) + // case Select(Select(a, b), stripMargin) => + // unfinishedStringStripMargin(b) + case Select(Apply(a, a2), b) if b.toString == "stripMargin" => + mapTreeList(a2) + case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => + val stripper = stripMargin.head.toString.charAt(1) + mapTreeList(a2, stripper) + case _ => + "unmatched in annotationToStrings: " + i.toString() + } + }) + } + values + } + + val annots = tpe.annotations + .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) + .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) + + Expr(annots) + def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { import quotes.reflect.* val tpe = TypeRepr.of[T].typeSymbol diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index 1804ec1ba..2b7a88d6e 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -130,8 +130,7 @@ object ParameterSchema { // name is e.g. "io.viash.config.Config.name", only keep "name" // name can also be "__this__" // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. - // val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) - val nameOverride = Option.empty // TODO + val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) val nameFromClass = name.split('.').last val name_ = (nameOverride, nameFromClass) match { case (Some(_), "__this__") => "__this__" From 343ecdef08cc7970c8342eb9ee3d9c9a0e90bcea Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 7 Nov 2024 12:25:26 +0100 Subject: [PATCH 28/46] add member annotation reflection and create the full annotation structures --- .../scala/io/viash/helpers/Mirroring.scala | 72 ++++++++++++++ .../io/viash/schemas/CollectedSchemas.scala | 95 +++---------------- .../io/viash/schemas/ParameterSchema.scala | 67 +++---------- 3 files changed, 98 insertions(+), 136 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index e1a9b7d6f..3845d30a1 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -30,6 +30,8 @@ inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } inline def annotationsOf[T]: List[(String, List[String])] = ${ annotationsOfImpl[T] } inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } +inline def memberAnnotationsOf[T]: List[(String, List[(String, List[String])])] = ${ memberAnnotationsOfImpl[T] } +inline def historyOf[T]: List[String] = ${ historyOfImpl[T] } def typeOfImpl[T: Type](using Quotes): Expr[String] = import quotes.reflect.* @@ -191,3 +193,73 @@ def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { val memberSymbols = tpe.fieldMembers.map(_.name) Expr(memberSymbols) } + +def memberAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[(String, List[String])])]] = { + import quotes.reflect.* + val tpe = TypeRepr.of[T].typeSymbol + val memberSymbols = tpe.fieldMembers + + def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = { + s.replaceAll("\\\\n", "\n").stripMargin(marginChar) + } + + def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { + l.map(i => i match { + // case Literal(Constant(value: String)) => + // unfinishedStringStripMargin(value, marginChar) + case Literal(value) => + unfinishedStringStripMargin(value.show(using Printer.ConstantCode), marginChar) + case _ => + "unmatched in mapTreeList: " + i.toString() + }).mkString + } + + // Traverse tree information and extract values or lists of values + def annotationToStrings(ann: Term): List[String] = { + // val name = ann.tree.tpe.toString() + val values = ann match { + case Apply(c, args: List[Tree]) => + args.collect({ + case i: Tree => + i match { + // Here 'Apply' contains lists + // While 'Select' has a single element + // case Literal(Constant(value: String)) => + // value + case Literal(value) => + value.show(using Printer.ConstantCode) + // case Select(Select(a, b), stripMargin) => + // unfinishedStringStripMargin(b) + case Select(Apply(a, a2), b) if b.toString == "stripMargin" => + mapTreeList(a2) + case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => + val stripper = stripMargin.head.toString.charAt(1) + mapTreeList(a2, stripper) + case _ => + "unmatched in annotationToStrings: " + i.toString() + } + }) + } + values + } + + val annots = + memberSymbols + .map{ case m => + val name = m.name + val n: Symbol = m + val annotations = m.annotations + .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) + .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) + (name, annotations) + } + + Expr(annots) +} + +def historyOfImpl[T: Type](using Quotes): Expr[List[String]] = { + import quotes.reflect.* + val baseClasses = TypeRepr.of[T].baseClasses.map(_.fullName).filter(_.startsWith("io.viash")) + + Expr(baseClasses) +} \ No newline at end of file diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 3f632d984..142409bff 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -84,62 +84,19 @@ object CollectedSchemas { private implicit val encodeDeprecatedOrRemoved: Encoder.AsObject[DeprecatedOrRemovedSchema] = deriveConfiguredEncoder private implicit val encodeExample: Encoder.AsObject[ExampleSchema] = deriveConfiguredEncoder - private inline final def summonLabels[T <: Tuple]: List[String] = - inline erasedValue[T] match - case _: EmptyTuple => Nil - case _: (t *: ts) => constValue[t].asInstanceOf[String] :: summonLabels[ts] - - private inline def getMembers[T /*TypeTag*/]/*(using mirror: Mirror.Of[T])*/(): (Map[String,List[MemberInfo]], List[Symbol]) = { - - // val name: String = constValue[mirror.MirroredLabel] - val name: String = typeOf[T] - - // Get all members and filter for constructors, first one should be the best (most complete) one - // Traits don't have constructors - // Get all parameters and store their short name - // val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String]) - // val constructorMembers = summonLabels[mirror.MirroredElemLabels] - val constructorMembers = fieldsOf[T] - - // val baseClasses = typeOf[T].baseClasses - // .filter(_.fullName.startsWith("io.viash")) - - // // If we're only getting a abstract class/trait, not a final implementation, use these definitions (otherwise we're left with nothing). - // val documentFully = - // baseClasses.length == 1 && - // baseClasses.head.isAbstract && - // baseClasses.head.annotations.exists(a => a.tree.tpe =:= typeOf[documentFully]) - - // val memberNames = typeOf[T].members - // .filter(!_.isMethod || documentFully) - // .map(_.shortName) - // .toSeq - val memberNames = membersOf[T] - - // val allMembers = baseClasses - // .zipWithIndex - // .flatMap{ case (baseClass, index) => - // baseClass.info.members - // .filter(_.fullName.startsWith("io.viash")) - // .filter(m => memberNames.contains(m.shortName)) - // .filter(m => !m.info.getClass.toString.endsWith("NullaryMethodType") || index != 0 || documentFully) // Only regular members if base class, otherwise all members - // .map(y => MemberInfo(y, (constructorMembers.contains(y.shortName)), baseClass.fullName, index)) - // } - // .groupBy(k => k.shortName) - - // (allMembers, baseClasses) - - // println(s"name: $name") - // println(s"constructorMembers: $constructorMembers") - // println(s"baseClasses: $baseClasses") - // println(s"documentFully: $documentFully") - // println(s"memberNames: $memberNames") - // println(s"allMembers: $allMembers") - - (Map.empty, Nil) + private def getMembers[T](): List[ParameterSchema] = { + val tpe = typeOf[T] + val history = historyOf[T] + val annotations = annotationsOf[T] + val thisMembers = ParameterSchema("__this__", tpe, history, annotations) + + val memberAnnotations = memberAnnotationsOf[T].map({ case (memberName, memberAnns) => + ParameterSchema(memberName, tpe, Nil, memberAnns) + }) + thisMembers +: memberAnnotations } - lazy val schemaClasses = List( + lazy val fullData = List( getMembers[Config](), getMembers[PackageConfig](), getMembers[BuildInfo](), @@ -223,32 +180,7 @@ object CollectedSchemas { .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") } - private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]): List[(String, String, List[String], List[Annotation])] = { - // val annMembers = members - // .map{ case (memberName, memberInfo) => { - // val h = memberInfo.head - // val annotations = memberInfo.flatMap(_.symbol.annotations) - // (h.fullName, h.symbol.info.toString, annotations, h.className, h.inheritanceIndex, Nil) - // } } - // .filter(_._3.length > 0) - // val annThis = ("__this__", classes.head.name.toString(), classes.head.annotations, "", 0, classes.map(_.fullName)) - // val allAnnotations = annThis :: annMembers.toList - // allAnnotations - // .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class - Nil - } - - private val getSchema: ((Map[String,List[MemberInfo]], List[Symbol])) => List[ParameterSchema] = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { - case (members, classes) => { - annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } - } - } - - // get all parameters for a given type, including parent class annotations - // def getParameters[T: TypeTag](): List[ParameterSchema] = getSchema(getMembers[T]()) - // Main call for documentation output - lazy val fullData: List[List[ParameterSchema]] = schemaClasses.map{ v => getSchema(v)} lazy val data: List[List[ParameterSchema]] = fullData.map(_.filter(p => !p.hasUndocumented && !p.hasInternalFunctionality)) def getKeyFromParamList(data: List[ParameterSchema]): String = data.find(p => p.name == "__this__").get.`type` @@ -272,8 +204,9 @@ object CollectedSchemas { // Main call for checking whether all arguments are annotated // Add extra non-annotated value so we can always somewhat check the code is functional - def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap { - v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) + def getAllNonAnnotated: Map[String, String] = (fullData :+ getMembers[CollectedSchemas]()).flatMap { + // v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) + _ => Nil }.toMap def getAllDeprecations: Map[String, DeprecatedOrRemovedSchema] = { diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index 2b7a88d6e..ddb88e292 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -40,49 +40,8 @@ final case class ParameterSchema( ) object ParameterSchema { - // Aid processing `augmentString` strings - private def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = { - s.replaceAll("\\\\n", "\n").stripMargin(marginChar) - } - // private def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { - // l.map(i => i match { - // case Literal(Constant(value: String)) => - // unfinishedStringStripMargin(value, marginChar) - // case _ => - // "unmatched in mapTreeList: " + i.toString() - // }).mkString - // } - - // Traverse tree information and extract values or lists of values - // private def annotationToStrings(ann: Annotation):(String, List[String]) = { - // val name = ann.tree.tpe.toString() - // val values = ann.tree match { - // case Apply(c, args: List[Tree]) => - // args.collect({ - // case i: Tree => - // i match { - // // Here 'Apply' contains lists - // // While 'Select' has a single element - // case Literal(Constant(value: String)) => - // value - // // case Select(Select(a, b), stripMargin) => - // // unfinishedStringStripMargin(b) - // case Select(Apply(a, a2), b) if b.toString == "stripMargin" => - // mapTreeList(a2) - // case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => - // val stripper = stripMargin.head.toString.charAt(1) - // mapTreeList(a2, stripper) - // case _ => - // "unmatched in annotationToStrings: " + i.toString() - // } - // }) - // } - // (name, values) - // } - - def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): ParameterSchema = { - println(s"ParameterSchema: $name, ${`type`}, $hierarchy, $annotations") + def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[(String, List[String])]): ParameterSchema = { def beautifyTypeName(s: String): String = { @@ -120,8 +79,6 @@ object ParameterSchema { } } - // val annStrings = annotations.map(annotationToStrings(_)) - val annStrings = List[(String, List[String])]() // TODO val hierarchyOption = hierarchy match { case l if l.length > 0 => Some(l) case _ => None @@ -130,7 +87,7 @@ object ParameterSchema { // name is e.g. "io.viash.config.Config.name", only keep "name" // name can also be "__this__" // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. - val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) + val nameOverride = annotations.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) val nameFromClass = name.split('.').last val name_ = (nameOverride, nameFromClass) match { case (Some(_), "__this__") => "__this__" @@ -143,26 +100,26 @@ object ParameterSchema { case (typeName, _, _) => typeName } - val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head}) - val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) - val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) + val description = annotations.collectFirst({case (name, value) if name.endsWith("description") => value.head}) + val example = annotations.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) + val exampleWithDescription = annotations.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) val examples = example ::: exampleWithDescription match { case l if l.length > 0 => Some(l) case _ => None } - val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head}) - val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) - val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) - val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head}) + val since = annotations.collectFirst({case (name, value) if name.endsWith("since") => value.head}) + val deprecated = annotations.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) + val removed = annotations.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) + val defaultFromAnnotation = annotations.collectFirst({case (name, value) if name.endsWith("default") => value.head}) val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") val default = defaultFromAnnotation orElse defaultFromType - val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { + val subclass = annotations.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { case l if l.nonEmpty => Some(l) case _ => None } - val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")} - val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")} + val undocumented = annotations.exists{ case (name, value) => name.endsWith("undocumented")} + val internalFunctionality = annotations.exists{ case (name, value) => name.endsWith("internalFunctionality")} ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass, undocumented, internalFunctionality) } From 7eb9a8f2d243fe2e3692b59b0fda8b67061ef5b4 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 14 Nov 2024 12:10:14 +0100 Subject: [PATCH 29/46] make getMembers inline so we don't lose the T type prune some more old code Add implementation for checking all non annotated Strip '"' from Literal values during reflecting --- .../scala/io/viash/helpers/Mirroring.scala | 16 +- .../io/viash/schemas/CollectedSchemas.scala | 203 ++++++++---------- .../scala/io/viash/schema/SchemaTest.scala | 10 +- 3 files changed, 105 insertions(+), 124 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 3845d30a1..0c2109218 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -30,7 +30,7 @@ inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } inline def annotationsOf[T]: List[(String, List[String])] = ${ annotationsOfImpl[T] } inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } -inline def memberAnnotationsOf[T]: List[(String, List[(String, List[String])])] = ${ memberAnnotationsOfImpl[T] } +inline def memberTypeAnnotationsOf[T]: List[(String, String, List[(String, List[String])])] = ${ memberTypeAnnotationsOfImpl[T] } inline def historyOf[T]: List[String] = ${ historyOfImpl[T] } def typeOfImpl[T: Type](using Quotes): Expr[String] = @@ -146,7 +146,7 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] // case Literal(Constant(value: String)) => // unfinishedStringStripMargin(value, marginChar) case Literal(value) => - unfinishedStringStripMargin(value.show(using Printer.ConstantCode), marginChar) + unfinishedStringStripMargin(value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\""), marginChar) case _ => "unmatched in mapTreeList: " + i.toString() }).mkString @@ -165,7 +165,7 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] // case Literal(Constant(value: String)) => // value case Literal(value) => - value.show(using Printer.ConstantCode) + value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") // case Select(Select(a, b), stripMargin) => // unfinishedStringStripMargin(b) case Select(Apply(a, a2), b) if b.toString == "stripMargin" => @@ -194,7 +194,7 @@ def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { Expr(memberSymbols) } -def memberAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[(String, List[String])])]] = { +def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, String, List[(String, List[String])])]] = { import quotes.reflect.* val tpe = TypeRepr.of[T].typeSymbol val memberSymbols = tpe.fieldMembers @@ -208,7 +208,7 @@ def memberAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[(Str // case Literal(Constant(value: String)) => // unfinishedStringStripMargin(value, marginChar) case Literal(value) => - unfinishedStringStripMargin(value.show(using Printer.ConstantCode), marginChar) + unfinishedStringStripMargin(value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\""), marginChar) case _ => "unmatched in mapTreeList: " + i.toString() }).mkString @@ -227,7 +227,7 @@ def memberAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[(Str // case Literal(Constant(value: String)) => // value case Literal(value) => - value.show(using Printer.ConstantCode) + value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") // case Select(Select(a, b), stripMargin) => // unfinishedStringStripMargin(b) case Select(Apply(a, a2), b) if b.toString == "stripMargin" => @@ -247,11 +247,11 @@ def memberAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[(Str memberSymbols .map{ case m => val name = m.name - val n: Symbol = m + val mTpe = "foo" val annotations = m.annotations .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) - (name, annotations) + (name, mTpe, annotations) } Expr(annots) diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 142409bff..d0df4cd77 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -60,21 +60,6 @@ final case class CollectedSchemas ( object CollectedSchemas { - - implicit class RichSymbol(s: Symbol) { - def shortName = ""//s.fullName.split('.').last - } - - case class MemberInfo ( - symbol: Symbol, - inConstructor: Boolean, - className: String, - inheritanceIndex: Int - ) { - def fullName = symbol.name//fullName - def shortName = symbol.shortName - } - private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) import io.viash.helpers.circe._ @@ -84,91 +69,97 @@ object CollectedSchemas { private implicit val encodeDeprecatedOrRemoved: Encoder.AsObject[DeprecatedOrRemovedSchema] = deriveConfiguredEncoder private implicit val encodeExample: Encoder.AsObject[ExampleSchema] = deriveConfiguredEncoder - private def getMembers[T](): List[ParameterSchema] = { + private inline def getMembers[T](): List[ParameterSchema] = { val tpe = typeOf[T] val history = historyOf[T] val annotations = annotationsOf[T] val thisMembers = ParameterSchema("__this__", tpe, history, annotations) - val memberAnnotations = memberAnnotationsOf[T].map({ case (memberName, memberAnns) => - ParameterSchema(memberName, tpe, Nil, memberAnns) + val memberAnnotations = memberTypeAnnotationsOf[T].map({ case (memberName, memberType, memberAnns) => + ParameterSchema(memberName, memberType, Nil, memberAnns) }) thisMembers +: memberAnnotations } - lazy val fullData = List( - getMembers[Config](), - getMembers[PackageConfig](), - getMembers[BuildInfo](), - getMembers[SysEnvTrait](), - - getMembers[Functionality](), - getMembers[Author](), - getMembers[ComputationalRequirements](), - getMembers[ArgumentGroup](), - getMembers[Links](), - getMembers[References](), - - getMembers[Runner](), - getMembers[ExecutableRunner](), - getMembers[NextflowRunner](), - - getMembers[Engine](), - getMembers[NativeEngine](), - getMembers[DockerEngine](), - - getMembers[Platform](), - getMembers[NativePlatform](), - getMembers[DockerPlatform](), - getMembers[NextflowPlatform](), - - getMembers[Requirements](), - getMembers[ApkRequirements](), - getMembers[AptRequirements](), - getMembers[DockerRequirements](), - getMembers[JavaScriptRequirements](), - getMembers[PythonRequirements](), - getMembers[RRequirements](), - getMembers[RubyRequirements](), - getMembers[YumRequirements](), - - getMembers[Argument[_]](), - getMembers[BooleanArgument](), - getMembers[BooleanTrueArgument](), - getMembers[BooleanFalseArgument](), - getMembers[DoubleArgument](), - getMembers[FileArgument](), - getMembers[IntegerArgument](), - getMembers[LongArgument](), - getMembers[StringArgument](), - - getMembers[Resource](), - getMembers[BashScript](), - getMembers[CSharpScript](), - getMembers[Executable](), - getMembers[JavaScriptScript](), - getMembers[NextflowScript](), - getMembers[PlainFile](), - getMembers[PythonScript](), - getMembers[RScript](), - getMembers[ScalaScript](), - - getMembers[NextflowDirectives](), - getMembers[NextflowAuto](), - getMembers[NextflowConfig](), - - getMembers[Dependency](), - getMembers[Repository](), - getMembers[LocalRepository](), - getMembers[GitRepository](), - getMembers[GithubRepository](), - getMembers[ViashhubRepository](), - getMembers[RepositoryWithName](), - getMembers[LocalRepositoryWithName](), - getMembers[GitRepositoryWithName](), - getMembers[GithubRepositoryWithName](), - getMembers[ViashhubRepositoryWithName](), - ) + // split the data in two parts to avoid the compiler complaining about the size + object memberData_part1 { + val part = List( + getMembers[Config](), + getMembers[PackageConfig](), + getMembers[BuildInfo](), + getMembers[SysEnvTrait](), + + getMembers[Functionality](), + getMembers[Author](), + getMembers[ComputationalRequirements](), + getMembers[ArgumentGroup](), + getMembers[Links](), + getMembers[References](), + + getMembers[Runner](), + getMembers[ExecutableRunner](), + getMembers[NextflowRunner](), + + getMembers[Engine](), + getMembers[NativeEngine](), + getMembers[DockerEngine](), + + getMembers[Platform](), + getMembers[NativePlatform](), + getMembers[DockerPlatform](), + getMembers[NextflowPlatform](), + + getMembers[Requirements](), + getMembers[ApkRequirements](), + getMembers[AptRequirements](), + getMembers[DockerRequirements](), + getMembers[JavaScriptRequirements](), + getMembers[PythonRequirements](), + getMembers[RRequirements](), + getMembers[RubyRequirements](), + getMembers[YumRequirements](), + + getMembers[Argument[_]](), + getMembers[BooleanArgument](), + getMembers[BooleanTrueArgument](), + getMembers[BooleanFalseArgument](), + getMembers[DoubleArgument](), + getMembers[FileArgument](), + getMembers[IntegerArgument](), + getMembers[LongArgument](), + getMembers[StringArgument](), + ) + } + object memberData_part2 { + val part = List( + getMembers[Resource](), + getMembers[BashScript](), + getMembers[CSharpScript](), + getMembers[Executable](), + getMembers[JavaScriptScript](), + getMembers[NextflowScript](), + getMembers[PlainFile](), + getMembers[PythonScript](), + getMembers[RScript](), + getMembers[ScalaScript](), + + getMembers[NextflowDirectives](), + getMembers[NextflowAuto](), + getMembers[NextflowConfig](), + + getMembers[Dependency](), + getMembers[Repository](), + getMembers[LocalRepository](), + getMembers[GitRepository](), + getMembers[GithubRepository](), + getMembers[ViashhubRepository](), + getMembers[RepositoryWithName](), + getMembers[LocalRepositoryWithName](), + getMembers[GitRepositoryWithName](), + getMembers[GithubRepositoryWithName](), + getMembers[ViashhubRepositoryWithName](), + ) + } private def trimTypeName(s: String) = { // first: io.viash.helpers.data_structures.OneOrMore[String] -> OneOrMore[String] @@ -180,6 +171,8 @@ object CollectedSchemas { .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") } + val fullData = memberData_part1.part ++ memberData_part2.part + // Main call for documentation output lazy val data: List[List[ParameterSchema]] = fullData.map(_.filter(p => !p.hasUndocumented && !p.hasInternalFunctionality)) @@ -187,27 +180,15 @@ object CollectedSchemas { def getJson: Json = data.asJson - private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = { - // val issueMembers = members - // .toList - // .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement. - // .map{ case (k, v) => (k, v.map(_.symbol.annotations.length).sum) } // (name, # annotations) - // .filter(_._2 == 0) - // .map(_._1) - - // val ownClassArr = if (classes.head.annotations.length == 0) Seq("__this__") else Nil - // issueMembers ++ ownClassArr - Nil - } - - def getMemberName(members: Map[String,List[MemberInfo]], classes: List[Symbol]): String = classes.head.shortName - - // Main call for checking whether all arguments are annotated + // Main call for checking whether all arguments are annotated with a description // Add extra non-annotated value so we can always somewhat check the code is functional - def getAllNonAnnotated: Map[String, String] = (fullData :+ getMembers[CollectedSchemas]()).flatMap { - // v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) - _ => Nil - }.toMap + def getAllNonAnnotated: List[(String, String)] = (data :+ getMembers[CollectedSchemas]()).flatMap { + members => { + val notAnnonated = members.filter(p => p.description == None) + val thisType = members.find(p => p.name == "__this__").get.`type` + notAnnonated.map(p => (thisType, p.name)) + } + } def getAllDeprecations: Map[String, DeprecatedOrRemovedSchema] = { val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap diff --git a/src/test/scala/io/viash/schema/SchemaTest.scala b/src/test/scala/io/viash/schema/SchemaTest.scala index 6fbbbc4fd..02db07448 100644 --- a/src/test/scala/io/viash/schema/SchemaTest.scala +++ b/src/test/scala/io/viash/schema/SchemaTest.scala @@ -33,14 +33,14 @@ class SchemaTest extends AnyFunSuite with BeforeAndAfterAll with PrivateMethodTe test("All schema class val members should be annotated") { val nonAnnotated = CollectedSchemas.getAllNonAnnotated - assert(nonAnnotated.contains("CollectedSchemas")) - assert(nonAnnotated("CollectedSchemas") == "__this__") + assert(nonAnnotated.contains(("CollectedSchemas", "__this__"))) - nonAnnotated.removed("CollectedSchemas").foreach { - case (key, member) => Console.err.println(s"$key - $member") + nonAnnotated.foreach { + case (key, member) if key != "CollectedSchemas" => Console.err.println(s"$key - $member") + case _ => () } - assert(nonAnnotated.size == 1) + assert(nonAnnotated.size == 9) // CollectedSchemas has 9 members, all of them unannotated } test("Check formatting of deprecation annotations") { From 2cd2130602a26711ad685381c0d4daafc7372d7b Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 15 Nov 2024 00:09:29 +0100 Subject: [PATCH 30/46] Get annotations of subclass fieldMembers too and flatten everything should do that for the class annotations too --- .../scala/io/viash/helpers/Mirroring.scala | 49 ++++++++++++------- .../scala/io/viash/schema/SchemaTest.scala | 2 +- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 0c2109218..1ce17dc25 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -29,7 +29,7 @@ inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctiona inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] } inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } inline def annotationsOf[T]: List[(String, List[String])] = ${ annotationsOfImpl[T] } -inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } +// inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } inline def memberTypeAnnotationsOf[T]: List[(String, String, List[(String, List[String])])] = ${ memberTypeAnnotationsOfImpl[T] } inline def historyOf[T]: List[String] = ${ historyOfImpl[T] } @@ -181,23 +181,22 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] values } + // TODO get annotations from base classes and flatten val annots = tpe.annotations .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) Expr(annots) -def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { - import quotes.reflect.* - val tpe = TypeRepr.of[T].typeSymbol - val memberSymbols = tpe.fieldMembers.map(_.name) - Expr(memberSymbols) -} +// def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { +// import quotes.reflect.* +// val tpe = TypeRepr.of[T].typeSymbol +// val memberSymbols = tpe.caseFields.map(_.name) +// Expr(memberSymbols) +// } def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, String, List[(String, List[String])])]] = { import quotes.reflect.* - val tpe = TypeRepr.of[T].typeSymbol - val memberSymbols = tpe.fieldMembers def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = { s.replaceAll("\\\\n", "\n").stripMargin(marginChar) @@ -243,18 +242,30 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin values } - val annots = - memberSymbols - .map{ case m => - val name = m.name - val mTpe = "foo" - val annotations = m.annotations - .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) - .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) - (name, mTpe, annotations) + val tpe = TypeRepr.of[T].typeSymbol + val baseClasses = TypeRepr.of[T].baseClasses.filter(_.fullName.startsWith("io.viash")) + + // base classes don't have case fields, so we need to get the member fields from the base classes and filter them + val caseFieldNames = tpe.caseFields.map(_.name) + + val annots = + baseClasses + .map{ case bc => + bc.fieldMembers + .filter(m => caseFieldNames.contains(m.name)) + .map(m => + val name = m.name + val mTpe = "foo" + val annotations = m.annotations + .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) + .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) + (name, mTpe, annotations) + ) } + // flatten the list of lists by name + val annotsFlattened = annots.flatten.groupBy(_._1).map{ case (k, v) => (k, v.head._2, v.flatMap(_._3)) }.toList - Expr(annots) + Expr(annotsFlattened) } def historyOfImpl[T: Type](using Quotes): Expr[List[String]] = { diff --git a/src/test/scala/io/viash/schema/SchemaTest.scala b/src/test/scala/io/viash/schema/SchemaTest.scala index 02db07448..81895e5bc 100644 --- a/src/test/scala/io/viash/schema/SchemaTest.scala +++ b/src/test/scala/io/viash/schema/SchemaTest.scala @@ -40,7 +40,7 @@ class SchemaTest extends AnyFunSuite with BeforeAndAfterAll with PrivateMethodTe case _ => () } - assert(nonAnnotated.size == 9) // CollectedSchemas has 9 members, all of them unannotated + assert(nonAnnotated.size == 10) // CollectedSchemas has 9 members, all of them unannotated + 1 __this__ member } test("Check formatting of deprecation annotations") { From 7639abafc615617dfe46ba8ebbe8f4c4f699e063 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 15 Nov 2024 08:07:06 +0100 Subject: [PATCH 31/46] strip unused code (after some tweaking to make it completely unused) --- .../io/viash/schemas/CollectedSchemas.scala | 20 +------------------ .../scala/io/viash/schema/SchemaTest.scala | 6 +++--- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index d0df4cd77..aa4aa6ad4 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -17,7 +17,6 @@ package io.viash.schemas -// import scala.reflect.runtime.universe._ import io.circe.{Encoder, Printer => JsonPrinter} import io.circe.syntax.EncoderOps @@ -26,12 +25,10 @@ import io.viash.runners._ import io.viash.engines._ import io.viash.platforms._ import io.circe.Json -import monocle.function.Cons import io.viash.config.Config import io.viash.config.BuildInfo import io.viash.packageConfig.PackageConfig import io.viash.helpers._ -import scala.collection.immutable.ListMap import io.viash.runners.nextflow.{NextflowConfig, NextflowAuto, NextflowDirectives} import io.viash.engines.requirements._ import io.viash.config.arguments._ @@ -42,29 +39,14 @@ import io.viash.config.Author import io.viash.config.ComputationalRequirements import io.viash.config.Links import io.viash.config.References -import scala.deriving.Mirror import scala.compiletime.{ codeOf, constValue, erasedValue, error, summonFrom, summonInline } import scala.annotation.Annotation -final case class CollectedSchemas ( - config: Map[String, List[ParameterSchema]], - functionality: Map[String, List[ParameterSchema]], - runners: Map[String, List[ParameterSchema]], - engines: Map[String, List[ParameterSchema]], - platforms: Map[String, List[ParameterSchema]], - requirements: Map[String, List[ParameterSchema]], - arguments: Map[String, List[ParameterSchema]], - resources: Map[String, List[ParameterSchema]], - nextflowParameters: Map[String, List[ParameterSchema]], -) - - object CollectedSchemas { private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) import io.viash.helpers.circe._ - private implicit val encodeConfigSchema: Encoder.AsObject[CollectedSchemas] = deriveConfiguredEncoder private implicit val encodeParameterSchema: Encoder.AsObject[ParameterSchema] = deriveConfiguredEncoderStrict private implicit val encodeDeprecatedOrRemoved: Encoder.AsObject[DeprecatedOrRemovedSchema] = deriveConfiguredEncoder private implicit val encodeExample: Encoder.AsObject[ExampleSchema] = deriveConfiguredEncoder @@ -182,7 +164,7 @@ object CollectedSchemas { // Main call for checking whether all arguments are annotated with a description // Add extra non-annotated value so we can always somewhat check the code is functional - def getAllNonAnnotated: List[(String, String)] = (data :+ getMembers[CollectedSchemas]()).flatMap { + def getAllNonAnnotated: List[(String, String)] = (data :+ getMembers[DeprecatedOrRemovedSchema]()).flatMap { members => { val notAnnonated = members.filter(p => p.description == None) val thisType = members.find(p => p.name == "__this__").get.`type` diff --git a/src/test/scala/io/viash/schema/SchemaTest.scala b/src/test/scala/io/viash/schema/SchemaTest.scala index 81895e5bc..f059e53b6 100644 --- a/src/test/scala/io/viash/schema/SchemaTest.scala +++ b/src/test/scala/io/viash/schema/SchemaTest.scala @@ -33,14 +33,14 @@ class SchemaTest extends AnyFunSuite with BeforeAndAfterAll with PrivateMethodTe test("All schema class val members should be annotated") { val nonAnnotated = CollectedSchemas.getAllNonAnnotated - assert(nonAnnotated.contains(("CollectedSchemas", "__this__"))) + assert(nonAnnotated.contains(("DeprecatedOrRemovedSchema", "__this__"))) nonAnnotated.foreach { - case (key, member) if key != "CollectedSchemas" => Console.err.println(s"$key - $member") + case (key, member) if key != "DeprecatedOrRemovedSchema" => Console.err.println(s"$key - $member") case _ => () } - assert(nonAnnotated.size == 10) // CollectedSchemas has 9 members, all of them unannotated + 1 __this__ member + assert(nonAnnotated.size == 4) // DeprecatedOrRemovedSchema has 3 members, all of them unannotated + 1 __this__ member } test("Check formatting of deprecation annotations") { From 77ba97acdc424ef7e5e785fe18867650a53a1285 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Mon, 18 Nov 2024 08:19:14 +0100 Subject: [PATCH 32/46] Improve which fields get annotated Fix comments why classes don't fully annotate base classes too --- src/main/scala/io/viash/helpers/Mirroring.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 1ce17dc25..e2ffea1ed 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -181,7 +181,8 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] values } - // TODO get annotations from base classes and flatten + // We're not adding annotations of base classes here. + // The base classes should be documented as well and the annotations will clash with the annotations of the specific class. val annots = tpe.annotations .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) @@ -246,13 +247,16 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin val baseClasses = TypeRepr.of[T].baseClasses.filter(_.fullName.startsWith("io.viash")) // base classes don't have case fields, so we need to get the member fields from the base classes and filter them + // only get the fields that are either case fields or have annotations val caseFieldNames = tpe.caseFields.map(_.name) + val annotatedFields = tpe.fieldMembers.filter(_.annotations.nonEmpty).map(_.name) + val toDocumentFields = (caseFieldNames ++ annotatedFields).distinct val annots = baseClasses .map{ case bc => bc.fieldMembers - .filter(m => caseFieldNames.contains(m.name)) + .filter(m => toDocumentFields.contains(m.name)) .map(m => val name = m.name val mTpe = "foo" From 900885214af600ef776e4091a0933460896c4b10 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 09:53:44 +0100 Subject: [PATCH 33/46] Fix type values of class members --- .../scala/io/viash/helpers/Mirroring.scala | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index e2ffea1ed..8d15eb814 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -243,13 +243,24 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin values } - val tpe = TypeRepr.of[T].typeSymbol - val baseClasses = TypeRepr.of[T].baseClasses.filter(_.fullName.startsWith("io.viash")) + // Use pattern matching to extract a simplified name + def simpleName(tpe: TypeRepr): String = tpe match { + case AppliedType(tycon, args) => + // If it's a type constructor with arguments, show it in a readable form + s"${simpleName(tycon)}[${args.map(simpleName).mkString(", ")}]" + case _ => + // Strip the full package name to get the simple type name + tpe.typeSymbol.name + } + + val tpe = TypeRepr.of[T] + val typeSymbol = tpe.typeSymbol + val baseClasses = tpe.baseClasses.filter(_.fullName.startsWith("io.viash")) // base classes don't have case fields, so we need to get the member fields from the base classes and filter them // only get the fields that are either case fields or have annotations - val caseFieldNames = tpe.caseFields.map(_.name) - val annotatedFields = tpe.fieldMembers.filter(_.annotations.nonEmpty).map(_.name) + val caseFieldNames = typeSymbol.caseFields.map(_.name) + val annotatedFields = typeSymbol.fieldMembers.filter(_.annotations.nonEmpty).map(_.name) val toDocumentFields = (caseFieldNames ++ annotatedFields).distinct val annots = @@ -259,7 +270,7 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin .filter(m => toDocumentFields.contains(m.name)) .map(m => val name = m.name - val mTpe = "foo" + val mTpe = simpleName(m.termRef.widen).replace(" ", "") val annotations = m.annotations .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) From 34736157b7f6ef7bdbda354ec992922e7eefbc3e Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 13:43:45 +0100 Subject: [PATCH 34/46] Tweak mirroring to output thing in the format we want, always stripMargin on all annotations Committing removal of the 'stripMargin' in the annotations in next commit --- src/main/scala/io/viash/helpers/Mirroring.scala | 16 +++++++++------- .../scala/io/viash/schemas/ParameterSchema.scala | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 8d15eb814..9bd69cccb 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -39,9 +39,9 @@ def typeOfImpl[T: Type](using Quotes): Expr[String] = // Use pattern matching to extract a simplified name def simpleName(tpe: TypeRepr): String = tpe match { - case AppliedType(tycon, args) => + case AppliedType(tycon, args) if !(args.length == 1 && args.head.typeSymbol.name == "Any") => // If it's a type constructor with arguments, show it in a readable form - s"${simpleName(tycon)}[${args.map(simpleName).mkString(", ")}]" + s"${simpleName(tycon)}[${args.map(simpleName).mkString(",")}]" case _ => // Strip the full package name to get the simple type name tpe.typeSymbol.name @@ -165,7 +165,8 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] // case Literal(Constant(value: String)) => // value case Literal(value) => - value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") + value.value.toString().stripMargin + // value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") // case Select(Select(a, b), stripMargin) => // unfinishedStringStripMargin(b) case Select(Apply(a, a2), b) if b.toString == "stripMargin" => @@ -227,7 +228,8 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin // case Literal(Constant(value: String)) => // value case Literal(value) => - value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") + value.value.toString().stripMargin + // value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") // case Select(Select(a, b), stripMargin) => // unfinishedStringStripMargin(b) case Select(Apply(a, a2), b) if b.toString == "stripMargin" => @@ -245,9 +247,9 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin // Use pattern matching to extract a simplified name def simpleName(tpe: TypeRepr): String = tpe match { - case AppliedType(tycon, args) => + case AppliedType(tycon, args) if !(args.length == 1 && args.head.typeSymbol.name == "Any") => // If it's a type constructor with arguments, show it in a readable form - s"${simpleName(tycon)}[${args.map(simpleName).mkString(", ")}]" + s"${simpleName(tycon)}[${args.map(simpleName).mkString(",")}]" case _ => // Strip the full package name to get the simple type name tpe.typeSymbol.name @@ -270,7 +272,7 @@ def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, Strin .filter(m => toDocumentFields.contains(m.name)) .map(m => val name = m.name - val mTpe = simpleName(m.termRef.widen).replace(" ", "") + val mTpe = simpleName(m.termRef.widen) val annotations = m.annotations .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash")) .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann))) diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index ddb88e292..f799969d5 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -114,7 +114,7 @@ object ParameterSchema { val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") val default = defaultFromAnnotation orElse defaultFromType val subclass = annotations.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { - case l if l.nonEmpty => Some(l) + case l if l.nonEmpty => Some(l.sorted()) case _ => None } From 583f144e71a05feaafde569981cfcd423dd5525c Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 13:47:55 +0100 Subject: [PATCH 35/46] remove .stripMargin from annotations. Regardless of being in comments already or not --- .../scala/io/viash/config/ArgumentGroup.scala | 8 +- src/main/scala/io/viash/config/Author.scala | 4 +- .../config/ComputationalRequirements.scala | 2 +- src/main/scala/io/viash/config/Config.scala | 54 +++++++------- src/main/scala/io/viash/config/Links.scala | 2 +- .../scala/io/viash/config/References.scala | 4 +- .../io/viash/config/arguments/Argument.scala | 4 +- .../config/arguments/BooleanArgument.scala | 34 ++++----- .../config/arguments/DoubleArgument.scala | 22 +++--- .../viash/config/arguments/FileArgument.scala | 31 ++++---- .../config/arguments/IntegerArgument.scala | 24 +++--- .../viash/config/arguments/LongArgument.scala | 24 +++--- .../config/arguments/StringArgument.scala | 20 ++--- .../config/dependencies/Dependency.scala | 12 +-- .../config/dependencies/GitRepository.scala | 4 +- .../dependencies/GitRepositoryWithName.scala | 4 +- .../dependencies/GithubRepository.scala | 4 +- .../GithubRepositoryWithName.scala | 4 +- .../config/dependencies/LocalRepository.scala | 4 +- .../LocalRepositoryWithName.scala | 4 +- .../config/dependencies/Repository.scala | 6 +- .../dependencies/ViashhubRepository.scala | 6 +- .../ViashhubRepositoryWithName.scala | 6 +- .../viash/config/resources/BashScript.scala | 2 +- .../viash/config/resources/CSharpScript.scala | 2 +- .../config/resources/JavaScriptScript.scala | 2 +- .../config/resources/NextflowScript.scala | 2 +- .../viash/config/resources/PythonScript.scala | 2 +- .../io/viash/config/resources/RScript.scala | 2 +- .../io/viash/config/resources/Resource.scala | 4 +- .../viash/config/resources/ScalaScript.scala | 2 +- .../scala/io/viash/engines/DockerEngine.scala | 6 +- src/main/scala/io/viash/engines/Engine.scala | 4 +- .../scala/io/viash/engines/NativeEngine.scala | 4 +- .../requirements/ApkRequirements.scala | 2 +- .../requirements/AptRequirements.scala | 2 +- .../requirements/DockerRequirements.scala | 14 ++-- .../requirements/JavaScriptRequirements.scala | 2 +- .../requirements/PythonRequirements.scala | 4 +- .../engines/requirements/RRequirements.scala | 4 +- .../engines/requirements/Requirements.scala | 2 +- .../requirements/RubyRequirements.scala | 2 +- .../requirements/YumRequirements.scala | 2 +- .../viash/functionality/Functionality.scala | 74 +++++++++---------- src/main/scala/io/viash/helpers/SysEnv.scala | 4 +- .../viash/packageConfig/PackageConfig.scala | 14 ++-- .../io/viash/platforms/DockerPlatform.scala | 44 +++++------ .../io/viash/platforms/NativePlatform.scala | 4 +- .../io/viash/platforms/NextflowPlatform.scala | 14 ++-- .../scala/io/viash/platforms/Platform.scala | 4 +- .../io/viash/runners/ExecutableRunner.scala | 42 +++++------ .../io/viash/runners/NextflowRunner.scala | 14 ++-- src/main/scala/io/viash/runners/Runner.scala | 4 +- .../viash/runners/nextflow/NextflowAuto.scala | 8 +- .../runners/nextflow/NextflowConfig.scala | 12 +-- .../runners/nextflow/NextflowDirectives.scala | 62 ++++++++-------- 56 files changed, 327 insertions(+), 326 deletions(-) diff --git a/src/main/scala/io/viash/config/ArgumentGroup.scala b/src/main/scala/io/viash/config/ArgumentGroup.scala index 9fd30d2ad..a311268f4 100644 --- a/src/main/scala/io/viash/config/ArgumentGroup.scala +++ b/src/main/scala/io/viash/config/ArgumentGroup.scala @@ -40,7 +40,7 @@ import io.viash.schemas._ | - name: "--output_optional" | type: file | direction: output - |""".stripMargin, + |""", "yaml") case class ArgumentGroup( @description("The name of the argument group.") @@ -62,7 +62,7 @@ case class ArgumentGroup( @example( """description: | | A (multiline) description of the purpose of the arguments - | in this argument group."""/*.stripMargin*/, "yaml") + | in this argument group.""", "yaml") @default("Empty") description: Option[String] = None, @@ -76,7 +76,7 @@ case class ArgumentGroup( | - @[boolean](arg_boolean) | - @[boolean_true](arg_boolean_true) | - @[boolean_false](arg_boolean_false) - |"""/*.stripMargin*/) + |""") @example( """arguments: | - name: --foo @@ -91,7 +91,7 @@ case class ArgumentGroup( | multiple_sep: ";" | - name: --bar | type: string - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") arguments: List[Argument[_]] = Nil diff --git a/src/main/scala/io/viash/config/Author.scala b/src/main/scala/io/viash/config/Author.scala index 6a1f20443..b07e9d0a7 100644 --- a/src/main/scala/io/viash/config/Author.scala +++ b/src/main/scala/io/viash/config/Author.scala @@ -34,7 +34,7 @@ import io.viash.schemas._ | twitter: janedoe | orcid: XXAABBCCXX | groups: [ one, two, three ] - |""".stripMargin, "yaml") + |""", "yaml") case class Author( @description("Full name of the author, usually in the name of FirstName MiddleName LastName.") name: String, @@ -48,7 +48,7 @@ case class Author( |* `"author"`: Authors who have made substantial contributions to the component. |* `"maintainer"`: The maintainer of the component. |* `"contributor"`: Authors who have made smaller contributions (such as code patches etc.). - |""".stripMargin) + |""") @default("Empty") roles: OneOrMore[String] = Nil, diff --git a/src/main/scala/io/viash/config/ComputationalRequirements.scala b/src/main/scala/io/viash/config/ComputationalRequirements.scala index 07efb2be2..29cef36ee 100644 --- a/src/main/scala/io/viash/config/ComputationalRequirements.scala +++ b/src/main/scala/io/viash/config/ComputationalRequirements.scala @@ -24,7 +24,7 @@ import io.viash.schemas._ """requirements: | cpus: 5 | memory: 10GB - |""".stripMargin, + |""", "yaml") @since("Viash 0.6.0") case class ComputationalRequirements( diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index 6319ed254..c5e9b2115 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -52,7 +52,7 @@ import io.viash.helpers.data_structures.oneOrMoreToList @description( """A Viash configuration is a YAML file which contains metadata to describe the behaviour and build target(s) of a component. |We commonly name this file `config.vsh.yaml` in our examples, but you can name it however you choose. - |""".stripMargin) + |""") @example( """name: hello_world |arguments: @@ -68,7 +68,7 @@ import io.viash.helpers.data_structures.oneOrMoreToList |engines: | - type: docker | image: "bash:4.0" - |""".stripMargin, "yaml") + |""", "yaml") case class Config( @description("Name of the component and the filename of the executable when built with `viash build`.") @example("name: this_is_my_component", "yaml") @@ -97,7 +97,7 @@ case class Config( || funder | fnd | for persons or organizations that furnished financial support for the development of the software | |The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive. - |"""/*.stripMargin*/) + |""") @example( """authors: | - name: Jane Doe @@ -111,7 +111,7 @@ case class Config( | - name: Tim Farbe | roles: [author] | email: tim@far.be - |"""/*.stripMargin*/, "yaml") + |""", "yaml") @since("Viash 0.3.1") @default("Empty") authors: List[Author] = Nil, @@ -123,7 +123,7 @@ case class Config( | - `description: Description of foo`, a description of the argument group. Multiline descriptions are supported. | - `arguments: [arg1, arg2, ...]`, list of the arguments. | - |"""/*.stripMargin*/) + |""") @example( """argument_groups: | - name: "Input" @@ -143,7 +143,7 @@ case class Config( | - name: "--output_optional" | type: file | direction: output - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription( """component_name @@ -161,7 +161,7 @@ case class Config( | | --optional_output | type: file - |"""/*.stripMargin*/, + |""", "bash", "This results in the following output when calling the component with the `--help` argument:") @since("Viash 0.5.14") @@ -178,14 +178,14 @@ case class Config( | * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`. | * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`. | * is_executable: `true` / `false`, whether the resulting resource file should be made executable. - |"""/*.stripMargin*/) + |""") @example( """resources: | - type: r_script | path: script.R | - type: file | path: resource1.txt - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") resources: List[Resource] = Nil, @@ -208,7 +208,7 @@ case class Config( """description: | | This component performs function Y and Z. | It is possible to make this a multiline string. - |"""/*.stripMargin*/, + |""", "yaml") description: Option[String] = None, @@ -224,7 +224,7 @@ case class Config( | - type: r_script | path: tests/test2.R | - path: resource1.txt - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") test_resources: List[Resource] = Nil, @@ -233,7 +233,7 @@ case class Config( @example( """info: | twitter: wizzkid - | classes: [ one, two, three ]"""/*.stripMargin*/, "yaml") + | classes: [ one, two, three ]""", "yaml") @since("Viash 0.4.0") @default("Empty") info: Json = Json.Null, @@ -247,12 +247,12 @@ case class Config( """@[Computational requirements](computational_requirements) related to running the component. |`cpus` specifies the maximum number of (logical) cpus a component is allowed to use., whereas |`memory` specifies the maximum amount of memory a component is allowed to allicate. Memory units must be - |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base)."""/*.stripMargin*/) + |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""") @example( """requirements: | cpus: 5 | memory: 10GB - |"""/*.stripMargin*/, + |""", "yaml") @since("Viash 0.6.0") @default("Empty") @@ -266,21 +266,21 @@ case class Config( | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |"""/*.stripMargin*/, + |""", "yaml", "Full specification of a repository") @exampleWithDescription( """dependencies: | - name: qc/multiqc | repository: "github://openpipelines-bio/modules:0.3.0" - |"""/*.stripMargin*/, + |""", "yaml", "Full specification of a repository using sugar syntax") @exampleWithDescription( """dependencies: | - name: qc/multiqc | repository: "openpipelines-bio" - |"""/*.stripMargin*/, + |""", "yaml", "Reference to a repository fully specified under 'repositories'") @default("Empty") @@ -288,14 +288,14 @@ case class Config( @description( """(Pre-)defines repositories that can be used as repository in dependencies. - |Allows reusing repository definitions in case it is used in multiple dependencies."""/*.stripMargin*/) + |Allows reusing repository definitions in case it is used in multiple dependencies.""") @example( """repositories: | - name: openpipelines-bio | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") repositories: List[RepositoryWithName] = Nil, @@ -323,7 +323,7 @@ case class Config( | journal={Baz}, | year={2024} | } - |"""/*.stripMargin*/, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") references: References = References(), @@ -336,7 +336,7 @@ case class Config( | homepage: "https://viash.io" | documentation: "https://viash.io/reference/" | issue_tracker: "https://github.com/viash-io/viash/issues" - |"""/*.stripMargin*/, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") links: Links = Links(), @@ -353,7 +353,7 @@ case class Config( | | - @[ExecutableRunner](executable_runner) | - @[NextflowRunner](nextflow_runner) - |"""/*.stripMargin*/) + |""") @since("Viash 0.8.0") @default("Empty") runners: List[Runner] = Nil, @@ -362,7 +362,7 @@ case class Config( | | - @[NativeEngine](native_engine) | - @[DockerEngine](docker_engine) - |"""/*.stripMargin*/) + |""") @since("Viash 0.8.0") @default("Empty") engines: List[Engine] = Nil, @@ -378,7 +378,7 @@ case class Config( @description( """The @[functionality](functionality) describes the behaviour of the script in terms of arguments and resources. |By specifying a few restrictions (e.g. mandatory arguments) and adding some descriptions, Viash will automatically generate a stylish command-line interface for you. - |""".stripMargin) + |""") @deprecated("Functionality level is deprecated, all functionality fields are now located on the top level of the config file.", "0.9.0", "0.10.0") @default("") private val functionality: Functionality = Functionality("foo") @@ -393,7 +393,7 @@ case class Config( | - @[boolean](arg_boolean) | - @[boolean_true](arg_boolean_true) | - @[boolean_false](arg_boolean_false) - |""".stripMargin) + |""") @default("Empty") private val arguments: List[Argument[_]] = Nil @@ -401,7 +401,7 @@ case class Config( """Config inheritance by including YAML partials. This is useful for defining common APIs in |separate files. `__merge__` can be used in any level of the YAML. For example, |not just in the config but also in the argument_groups or any of the engines. - |""".stripMargin) + |""") @example("__merge__: ../api/common_interface.yaml", "yaml") @since("Viash 0.6.3") private val `__merge__`: Option[File] = None @@ -412,7 +412,7 @@ case class Config( | - @[Native](platform_native) | - @[Docker](platform_docker) | - @[Nextflow](platform_nextflow) - |""".stripMargin) + |""") @default("Empty") @deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0") private val platforms: List[Platform] = Nil diff --git a/src/main/scala/io/viash/config/Links.scala b/src/main/scala/io/viash/config/Links.scala index 95cc0fda5..6f3375470 100644 --- a/src/main/scala/io/viash/config/Links.scala +++ b/src/main/scala/io/viash/config/Links.scala @@ -27,7 +27,7 @@ import io.viash.schemas._ | homepage: "https://viash.io" | documentation: "https://viash.io/reference/" | issue_tracker: "https://github.com/viash-io/viash/issues" - |""".stripMargin, "yaml") + |""", "yaml") @since("Viash 0.9.0") case class Links( @description("Source repository url.") diff --git a/src/main/scala/io/viash/config/References.scala b/src/main/scala/io/viash/config/References.scala index ab63add56..6dbaa7285 100644 --- a/src/main/scala/io/viash/config/References.scala +++ b/src/main/scala/io/viash/config/References.scala @@ -31,7 +31,7 @@ import io.viash.helpers.data_structures.{OneOrMore, listToOneOrMore} | journal={Baz}, | year={2024} | } - |""".stripMargin, "yaml") + |""", "yaml") @since("Viash 0.9.0") case class References( @description("One or multiple DOI reference(s) of the component.") @@ -48,7 +48,7 @@ case class References( | journal={Baz}, | year={2024} | } - |"""/*.stripMargin*/, "yaml") + |""", "yaml") @default("Empty") bibtex: OneOrMore[String] = Nil, ) diff --git a/src/main/scala/io/viash/config/arguments/Argument.scala b/src/main/scala/io/viash/config/arguments/Argument.scala index 869da0e33..d95f5bcf5 100644 --- a/src/main/scala/io/viash/config/arguments/Argument.scala +++ b/src/main/scala/io/viash/config/arguments/Argument.scala @@ -32,7 +32,7 @@ import java.nio.file.Paths | - @[boolean](arg_boolean) | - @[boolean_true](arg_boolean_true) | - @[boolean_false](arg_boolean_false) - |""".stripMargin) + |""") @example( """arguments: | - name: --foo @@ -47,7 +47,7 @@ import java.nio.file.Paths | multiple_sep: ";" | - name: --bar | type: string - |""".stripMargin, + |""", "yaml") @subclass("BooleanArgument") @subclass("BooleanTrueArgument") diff --git a/src/main/scala/io/viash/config/arguments/BooleanArgument.scala b/src/main/scala/io/viash/config/arguments/BooleanArgument.scala index 9835de12c..3f8542773 100644 --- a/src/main/scala/io/viash/config/arguments/BooleanArgument.scala +++ b/src/main/scala/io/viash/config/arguments/BooleanArgument.scala @@ -33,7 +33,7 @@ abstract class BooleanArgumentBase extends Argument[Boolean] { | default: true | description: Trim whitespace from the final output | alternatives: ["-t"] - |""".stripMargin, + |""", "yaml") @subclass("boolean") case class BooleanArgument( @@ -43,7 +43,7 @@ case class BooleanArgument( | - `--trim` is a long option, which can be passed with `executable_name --trim` | - `-t` is a short option, which can be passed with `executable_name -t` | - `trim` is an argument, which can be passed with `executable_name trim` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -66,7 +66,7 @@ case class BooleanArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -74,7 +74,7 @@ case class BooleanArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -84,7 +84,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | example: true - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") example: OneOrMore[Boolean] = Nil, @@ -94,7 +94,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | default: true - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") default: OneOrMore[Boolean] = Nil, @@ -104,7 +104,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | required: true - |"""/*.stripMargin*/, + |""", "yaml") @default("False") required: Boolean = false, @@ -117,7 +117,7 @@ case class BooleanArgument( """- name: --my_boolean | type: boolean | multiple: true - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_boolean=true:true:false", "bash", "Here's an example of how to use this:") @default("False") @@ -129,7 +129,7 @@ case class BooleanArgument( | type: boolean | multiple: true | multiple_sep: ";" - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_boolean=true,true,false", "bash", "Here's an example of how to use this:") @default(";") @@ -169,7 +169,7 @@ case class BooleanArgument( | type: boolean_true | description: Ignore console output | alternatives: ["-s"] - |""".stripMargin, + |""", "yaml") @subclass("boolean_true") case class BooleanTrueArgument( @@ -179,7 +179,7 @@ case class BooleanTrueArgument( | - `--silent` is a long option, which can be passed with `executable_name --silent` | - `-s` is a short option, which can be passed with `executable_name -s` | - `silent` is an argument, which can be passed with `executable_name silent` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -202,7 +202,7 @@ case class BooleanTrueArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -210,7 +210,7 @@ case class BooleanTrueArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -261,7 +261,7 @@ case class BooleanTrueArgument( | type: boolean_false | description: Disable logging | alternatives: ["-nl"] - |""".stripMargin, + |""", "yaml") @subclass("boolean_false") case class BooleanFalseArgument( @@ -271,7 +271,7 @@ case class BooleanFalseArgument( | - `--no-log` is a long option, which can be passed with `executable_name --no-log` | - `-n` is a short option, which can be passed with `executable_name -n` | - `no-log` is an argument, which can be passed with `executable_name no-log` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -294,7 +294,7 @@ case class BooleanFalseArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -302,7 +302,7 @@ case class BooleanFalseArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, diff --git a/src/main/scala/io/viash/config/arguments/DoubleArgument.scala b/src/main/scala/io/viash/config/arguments/DoubleArgument.scala index 273818e83..5a5c2f4e8 100644 --- a/src/main/scala/io/viash/config/arguments/DoubleArgument.scala +++ b/src/main/scala/io/viash/config/arguments/DoubleArgument.scala @@ -29,7 +29,7 @@ import io.viash.schemas._ | default: 1.5 | description: Litres of fluid to process | alternatives: ["-l"] - |""".stripMargin, + |""", "yaml") @subclass("double") case class DoubleArgument( @@ -39,7 +39,7 @@ case class DoubleArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -62,7 +62,7 @@ case class DoubleArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -70,7 +70,7 @@ case class DoubleArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -80,7 +80,7 @@ case class DoubleArgument( """- name: --my_double | type: double | example: 5.8 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") example: OneOrMore[Double] = Nil, @@ -90,7 +90,7 @@ case class DoubleArgument( """- name: --my_double | type: double | default: 5.8 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") default: OneOrMore[Double] = Nil, @@ -100,7 +100,7 @@ case class DoubleArgument( """- name: --my_double | type: double | required: true - |"""/*.stripMargin*/, + |""", "yaml") @default("False") required: Boolean = false, @@ -110,7 +110,7 @@ case class DoubleArgument( """- name: --my_double | type: double | min: 25.5 - |"""/*.stripMargin*/, + |""", "yaml") min: Option[Double] = None, @@ -119,7 +119,7 @@ case class DoubleArgument( """- name: --my_double | type: double | max: 80.4 - |"""/*.stripMargin*/, + |""", "yaml") max: Option[Double] = None, @@ -131,7 +131,7 @@ case class DoubleArgument( """- name: --my_double | type: double | multiple: true - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_double=5.8:22.6:200.4", "bash", "Here's an example of how to use this:") @default("False") @@ -143,7 +143,7 @@ case class DoubleArgument( | type: double | multiple: true | multiple_sep: ";" - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_double=5.8,22.6,200.4", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/FileArgument.scala b/src/main/scala/io/viash/config/arguments/FileArgument.scala index 6216d80b2..052ab803b 100644 --- a/src/main/scala/io/viash/config/arguments/FileArgument.scala +++ b/src/main/scala/io/viash/config/arguments/FileArgument.scala @@ -30,7 +30,7 @@ import io.viash.schemas._ | must_exist: true | description: CSV file to read contents from | alternatives: ["-i"] - |""".stripMargin, + |""", "yaml") @subclass("file") case class FileArgument( @@ -40,7 +40,7 @@ case class FileArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -63,7 +63,7 @@ case class FileArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -71,7 +71,7 @@ case class FileArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -81,7 +81,7 @@ case class FileArgument( """- name: --my_file | type: file | example: data.csv - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") example: OneOrMore[Path] = Nil, @@ -91,18 +91,19 @@ case class FileArgument( """- name: --my_file | type: file | default: data.csv - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") default: OneOrMore[Path] = Nil, - @description("Checks whether the file or folder exists. For input files, this check will happen " + - "before the execution of the script, while for output files the check will happen afterwards.") + @description( + """Checks whether the file or folder exists. For input files, this check will happen + |before the execution of the script, while for output files the check will happen afterwards.""") @example( """- name: --my_file | type: file | must_exist: true - |"""/*.stripMargin*/, + |""", "yaml") @default("True") must_exist: Boolean = true, @@ -113,7 +114,7 @@ case class FileArgument( | type: file | direction: output | create_parent: true - |"""/*.stripMargin*/, + |""", "yaml") @default("True") create_parent: Boolean = true, @@ -123,7 +124,7 @@ case class FileArgument( """- name: --my_file | type: file | required: true - |"""/*.stripMargin*/, + |""", "yaml") @default("False") required: Boolean = false, @@ -133,7 +134,7 @@ case class FileArgument( """- name: --my_output_file | type: file | direction: output - |"""/*.stripMargin*/, + |""", "yaml") @default("Input") direction: Direction = Input, @@ -152,12 +153,12 @@ case class FileArgument( | automatically attempt to expand the expression. | |Other output arguments (e.g. integer, double, ...) are not supported yet. - |"""/*.stripMargin*/) + |""") @example( """- name: --my_files | type: file | multiple: true - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_files=firstFile.csv:anotherFile.csv:yetAnother.csv", "bash", "Here's an example of how to use this:") @default("False") @@ -169,7 +170,7 @@ case class FileArgument( | type: file | multiple: true | multiple_sep: ";" - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_files=firstFile.csv,anotherFile.csv,yetAnother.csv", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/IntegerArgument.scala b/src/main/scala/io/viash/config/arguments/IntegerArgument.scala index 548d51cb8..7e9f22cda 100644 --- a/src/main/scala/io/viash/config/arguments/IntegerArgument.scala +++ b/src/main/scala/io/viash/config/arguments/IntegerArgument.scala @@ -29,7 +29,7 @@ import io.viash.schemas._ | default: 16 | description: Amount of CPU cores to use | alternatives: ["-c"] - |""".stripMargin, + |""", "yaml") @subclass("integer") case class IntegerArgument( @@ -39,7 +39,7 @@ case class IntegerArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -62,7 +62,7 @@ case class IntegerArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -70,7 +70,7 @@ case class IntegerArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -80,7 +80,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | example: 100 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") example: OneOrMore[Int] = Nil, @@ -90,7 +90,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | default: 100 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") default: OneOrMore[Int] = Nil, @@ -100,7 +100,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | required: true - |"""/*.stripMargin*/, + |""", "yaml") @default("False") required: Boolean = false, @@ -110,7 +110,7 @@ case class IntegerArgument( """- name: --values | type: integer | choices: [1024, 2048, 4096] - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") choices: List[Int] = Nil, @@ -120,7 +120,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | min: 50 - |"""/*.stripMargin*/, + |""", "yaml") min: Option[Int] = None, @@ -129,7 +129,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | max: 150 - |"""/*.stripMargin*/, + |""", "yaml") max: Option[Int] = None, @@ -141,7 +141,7 @@ case class IntegerArgument( """- name: --my_integer | type: integer | multiple: true - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:") @default("False") @@ -153,7 +153,7 @@ case class IntegerArgument( | type: integer | multiple: true | multiple_sep: ";" - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/LongArgument.scala b/src/main/scala/io/viash/config/arguments/LongArgument.scala index 92a7ffb2f..502b64a54 100644 --- a/src/main/scala/io/viash/config/arguments/LongArgument.scala +++ b/src/main/scala/io/viash/config/arguments/LongArgument.scala @@ -29,7 +29,7 @@ import io.viash.schemas._ | default: 16 | description: Amount of CPU cores to use | alternatives: ["-c"] - |""".stripMargin, + |""", "yaml") @since("Viash 0.6.1") @subclass("long") @@ -40,7 +40,7 @@ case class LongArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -63,7 +63,7 @@ case class LongArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -71,7 +71,7 @@ case class LongArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -81,7 +81,7 @@ case class LongArgument( """- name: --my_long | type: long | example: 100 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") example: OneOrMore[Long] = Nil, @@ -91,7 +91,7 @@ case class LongArgument( """- name: --my_long | type: long | default: 100 - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") default: OneOrMore[Long] = Nil, @@ -101,7 +101,7 @@ case class LongArgument( """- name: --my_long | type: long | required: true - |"""/*.stripMargin*/, + |""", "yaml") @default("False") required: Boolean = false, @@ -111,7 +111,7 @@ case class LongArgument( """- name: --values | type: long | choices: [1024, 2048, 4096] - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") choices: List[Long] = Nil, @@ -121,7 +121,7 @@ case class LongArgument( """- name: --my_long | type: long | min: 50 - |"""/*.stripMargin*/, + |""", "yaml") min: Option[Long] = None, @@ -130,7 +130,7 @@ case class LongArgument( """- name: --my_long | type: long | max: 150 - |"""/*.stripMargin*/, + |""", "yaml") max: Option[Long] = None, @@ -142,7 +142,7 @@ case class LongArgument( """- name: --my_long | type: long | multiple: true - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:") @default("False") @@ -154,7 +154,7 @@ case class LongArgument( | type: long | multiple: true | multiple_sep: ";" - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/arguments/StringArgument.scala b/src/main/scala/io/viash/config/arguments/StringArgument.scala index 133115990..530bbeca2 100644 --- a/src/main/scala/io/viash/config/arguments/StringArgument.scala +++ b/src/main/scala/io/viash/config/arguments/StringArgument.scala @@ -29,7 +29,7 @@ import io.viash.schemas._ | default: "meaning of life" | description: The term to search for | alternatives: ["-q"] - |""".stripMargin, + |""", "yaml") @subclass("string") case class StringArgument( @@ -39,7 +39,7 @@ case class StringArgument( | - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value` | - `-f` is a short option, which can be passed with `executable_name -f value` | - `foo` is an argument, which can be passed with `executable_name value` - |"""/*.stripMargin*/) + |""") name: String, @description("List of alternative format variations for this argument.") @@ -62,7 +62,7 @@ case class StringArgument( @example( """description: | | A (multiline) description of the purpose of - | this argument."""/*.stripMargin*/, "yaml") + | this argument.""", "yaml") @default("Empty") description: Option[String] = None, @@ -70,7 +70,7 @@ case class StringArgument( @example( """info: | category: cat1 - | labels: [one, two, three]"""/*.stripMargin*/, "yaml") + | labels: [one, two, three]""", "yaml") @since("Viash 0.6.3") @default("Empty") info: Json = Json.Null, @@ -80,7 +80,7 @@ case class StringArgument( """- name: --my_string | type: string | example: "Hello World" - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") example: OneOrMore[String] = Nil, @@ -90,7 +90,7 @@ case class StringArgument( """- name: --my_string | type: string | default: "The answer is 42" - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") default: OneOrMore[String] = Nil, @@ -100,7 +100,7 @@ case class StringArgument( """- name: --my_string | type: string | required: true - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") required: Boolean = false, @@ -110,7 +110,7 @@ case class StringArgument( """- name: --language | type: string | choices: ["python", "r", "javascript"] - |"""/*.stripMargin*/, + |""", "yaml") @default("Empty") choices: List[String] = Nil, @@ -123,7 +123,7 @@ case class StringArgument( """- name: --my_string | type: string | multiple: true - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_string=Marc:Susan:Paul", "bash", "Here's an example of how to use this:") @default("False") @@ -135,7 +135,7 @@ case class StringArgument( | type: string | multiple: true | multiple_sep: ";" - |"""/*.stripMargin*/, + |""", "yaml") @exampleWithDescription("my_component --my_string=Marc,Susan,Paul", "bash", "Here's an example of how to use this:") @default(";") diff --git a/src/main/scala/io/viash/config/dependencies/Dependency.scala b/src/main/scala/io/viash/config/dependencies/Dependency.scala index b1ea2f671..cfa0fd308 100644 --- a/src/main/scala/io/viash/config/dependencies/Dependency.scala +++ b/src/main/scala/io/viash/config/dependencies/Dependency.scala @@ -27,7 +27,7 @@ import io.viash.exceptions.MissingBuildYamlException @description( """Specifies a Viash component (script or executable) that should be made available for the code defined in the component. |The dependency components are collected and copied to the output folder during the Viash build step. - |""".stripMargin) + |""") @exampleWithDescription( """dependencies: | - name: qc/multiqc @@ -35,7 +35,7 @@ import io.viash.exceptions.MissingBuildYamlException | type: github | repo: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |""", "yaml", "Definition of dependency with a fully defined repository" ) @@ -43,7 +43,7 @@ import io.viash.exceptions.MissingBuildYamlException """dependencies: | - name: qc/multiqc | repository: "github://openpipelines-bio/modules:0.3.0" - |""".stripMargin, + |""", "yaml", "Definition of a dependency with a repository using sugar syntax." ) @@ -51,14 +51,14 @@ import io.viash.exceptions.MissingBuildYamlException """dependencies: | - name: qc/multiqc | repository: "openpipelines-bio" - |""".stripMargin, + |""", "yaml", "Definition of a dependency with a repository defined as 'openpipelines-bio' under `.repositories`." ) @exampleWithDescription( """dependencies: | - name: qc/multiqc - |""".stripMargin, + |""", "yaml", "Definition of a local dependency. This dependency is present in the current code base and will be built when `viash ns build` is run." ) @@ -76,7 +76,7 @@ case class Dependency( |This must either be a full definition of the repository or the name of a repository referenced as it is defined under repositories. |Additionally, the full definition can be specified as a single string where all parameters such as repository type, url, branch or tag are specified. |Omitting the value sets the dependency as a local dependency, ie. the dependency is available in the same namespace as the component. - |"""/*.stripMargin*/) + |""") @default("Empty") repository: Either[String, Repository] = Right(LocalRepository()), diff --git a/src/main/scala/io/viash/config/dependencies/GitRepository.scala b/src/main/scala/io/viash/config/dependencies/GitRepository.scala index f1d924f82..8a8c45413 100644 --- a/src/main/scala/io/viash/config/dependencies/GitRepository.scala +++ b/src/main/scala/io/viash/config/dependencies/GitRepository.scala @@ -28,7 +28,7 @@ import java.nio.file.Paths """type: git |uri: git+https://github.com/openpipelines-bio/openpipeline.git |tag: 0.8.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -36,7 +36,7 @@ import java.nio.file.Paths |uri: git+https://gitlab.com/viash-io/viash.git |tag: 0.7.1 |path: src/test/resources/testns - |""".stripMargin, + |""", "yaml" ) @subclass("git") diff --git a/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala index e8c1d0530..cbf24c383 100644 --- a/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala +++ b/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala @@ -29,7 +29,7 @@ import java.nio.file.Paths |type: git |uri: git+https://github.com/openpipelines-bio/openpipeline.git |tag: 0.8.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -38,7 +38,7 @@ import java.nio.file.Paths |uri: git+https://gitlab.com/viash-io/viash.git |tag: 0.7.1 |path: src/test/resources/testns - |""".stripMargin, + |""", "yaml" ) @subclass("gitwithname") diff --git a/src/main/scala/io/viash/config/dependencies/GithubRepository.scala b/src/main/scala/io/viash/config/dependencies/GithubRepository.scala index 016eaf7e0..30022c510 100644 --- a/src/main/scala/io/viash/config/dependencies/GithubRepository.scala +++ b/src/main/scala/io/viash/config/dependencies/GithubRepository.scala @@ -28,7 +28,7 @@ import java.nio.file.Paths """type: github |repo: openpipelines-bio/openpipeline |tag: 0.8.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -36,7 +36,7 @@ import java.nio.file.Paths |repo: viash-io/viash |tag: 0.7.1 |path: src/test/resources/testns - |""".stripMargin, + |""", "yaml" ) @subclass("github") diff --git a/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala index 80c7f1561..4167dd32d 100644 --- a/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala +++ b/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala @@ -29,7 +29,7 @@ import java.nio.file.Paths |type: github |repo: openpipelines-bio/openpipeline |tag: 0.8.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -38,7 +38,7 @@ import java.nio.file.Paths |repo: viash-io/viash |tag: 0.7.1 |path: src/test/resources/testns - |""".stripMargin, + |""", "yaml" ) @subclass("githubwithname") diff --git a/src/main/scala/io/viash/config/dependencies/LocalRepository.scala b/src/main/scala/io/viash/config/dependencies/LocalRepository.scala index 0475f7355..36f54fa4e 100644 --- a/src/main/scala/io/viash/config/dependencies/LocalRepository.scala +++ b/src/main/scala/io/viash/config/dependencies/LocalRepository.scala @@ -24,12 +24,12 @@ import java.nio.file.Paths """Defines a locally present and available repository. |This can be used to define components from the same code base as the current component. |Alternatively, this can be used to refer to a code repository present on the local hard-drive instead of fetchable remotely, for example during development. - |""".stripMargin + |""" ) @exampleWithDescription( """type: local |path: /additional_code/src - |""".stripMargin, + |""", "yaml", "Refer to a local code repository under `additional_code/src` referenced to the Viash Package Config file." ) diff --git a/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala index 9df0035c7..3c7bc1ac0 100644 --- a/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala +++ b/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala @@ -24,13 +24,13 @@ import java.nio.file.Paths """Defines a locally present and available repository. |This can be used to define components from the same code base as the current component. |Alternatively, this can be used to refer to a code repository present on the local hard-drive instead of fetchable remotely, for example during development. - |""".stripMargin + |""" ) @exampleWithDescription( """name: my_local_code |type: local |path: /additional_code/src - |""".stripMargin, + |""", "yaml", "Refer to a local code repository under `additional_code/src` referenced to the Viash Package Config file." ) diff --git a/src/main/scala/io/viash/config/dependencies/Repository.scala b/src/main/scala/io/viash/config/dependencies/Repository.scala index 5afbe1ce3..2d2c3dfdf 100644 --- a/src/main/scala/io/viash/config/dependencies/Repository.scala +++ b/src/main/scala/io/viash/config/dependencies/Repository.scala @@ -28,14 +28,14 @@ import java.nio.file.{Path, Paths, Files} | - @[git](repo_git) | - @[github](repo_github) | - @[vsh](repo_vsh) - |""".stripMargin) + |""") @exampleWithDescription( """repositories: | - name: openpipelines-bio | type: github | repo: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |""", "yaml", "Definition of a repository in the component config or package config.") @exampleWithDescription( @@ -45,7 +45,7 @@ import java.nio.file.{Path, Paths, Files} | type: github | repo: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |""", "yaml", "Definition of dependency with a fully defined repository") @subclass("LocalRepository") diff --git a/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala b/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala index 2e640502a..8272c649b 100644 --- a/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala +++ b/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala @@ -28,14 +28,14 @@ import java.nio.file.Paths """type: vsh |repo: biobox |tag: 0.1.0 - |""".stripMargin, + |""", "yaml" ) @example( """type: vsh |repo: openpipelines-bio/openpipeline |tag: 0.8.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -43,7 +43,7 @@ import java.nio.file.Paths |repo: openpipelines-bio/openpipeline |tag: 0.7.1 |path: src/test/resources/testns - |""".stripMargin, + |""", "yaml" ) @subclass("viashhub") diff --git a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala index 66acddf88..f83f68942 100644 --- a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala +++ b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala @@ -29,7 +29,7 @@ import java.nio.file.Paths |type: vsh |repo: biobox |tag: 0.1.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -37,7 +37,7 @@ import java.nio.file.Paths |type: vsh |repo: openpipelines-bio/openpipeline |tag: 0.8.0 - |""".stripMargin, + |""", "yaml" ) @example( @@ -46,7 +46,7 @@ import java.nio.file.Paths |repo: openpipelines-bio/openpipeline |tag: 0.7.1 |path: src/test/resources/testns - |""".stripMargin, + |""", "yaml" ) @subclass("viashhubwithname") diff --git a/src/main/scala/io/viash/config/resources/BashScript.scala b/src/main/scala/io/viash/config/resources/BashScript.scala index b5f8a088f..fc819a37b 100644 --- a/src/main/scala/io/viash/config/resources/BashScript.scala +++ b/src/main/scala/io/viash/config/resources/BashScript.scala @@ -27,7 +27,7 @@ import io.viash.config.Config @description("""An executable Bash script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. - |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin) + |When defined in test_resources, all entries will be executed during `viash test`.""") @subclass("bash_script") case class BashScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/config/resources/CSharpScript.scala b/src/main/scala/io/viash/config/resources/CSharpScript.scala index 17b8bc67f..2fc286060 100644 --- a/src/main/scala/io/viash/config/resources/CSharpScript.scala +++ b/src/main/scala/io/viash/config/resources/CSharpScript.scala @@ -27,7 +27,7 @@ import io.viash.config.Config @description("""An executable C# script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. - |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin) + |When defined in test_resources, all entries will be executed during `viash test`.""") @subclass("csharp_script") case class CSharpScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/config/resources/JavaScriptScript.scala b/src/main/scala/io/viash/config/resources/JavaScriptScript.scala index d11962ed9..b35fd7bb5 100644 --- a/src/main/scala/io/viash/config/resources/JavaScriptScript.scala +++ b/src/main/scala/io/viash/config/resources/JavaScriptScript.scala @@ -27,7 +27,7 @@ import io.viash.config.arguments.{Argument, StringArgument, IntegerArgument, Boo @description("""An executable JavaScript script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. - |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin) + |When defined in test_resources, all entries will be executed during `viash test`.""") @subclass("javascript_script") case class JavaScriptScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/config/resources/NextflowScript.scala b/src/main/scala/io/viash/config/resources/NextflowScript.scala index 819e23030..e41d733ed 100644 --- a/src/main/scala/io/viash/config/resources/NextflowScript.scala +++ b/src/main/scala/io/viash/config/resources/NextflowScript.scala @@ -30,7 +30,7 @@ import io.viash.helpers.circe._ import io.viash.ViashNamespace import io.viash.config.dependencies.Dependency -@description("""A Nextflow script. Work in progress; added mainly for annotation at the moment.""".stripMargin) +@description("""A Nextflow script. Work in progress; added mainly for annotation at the moment.""") @subclass("nextflow_script") case class NextflowScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/config/resources/PythonScript.scala b/src/main/scala/io/viash/config/resources/PythonScript.scala index 40bf1030c..feba17735 100644 --- a/src/main/scala/io/viash/config/resources/PythonScript.scala +++ b/src/main/scala/io/viash/config/resources/PythonScript.scala @@ -27,7 +27,7 @@ import io.viash.config.Config @description("""An executable Python script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. - |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin) + |When defined in test_resources, all entries will be executed during `viash test`.""") @subclass("python_script") case class PythonScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/config/resources/RScript.scala b/src/main/scala/io/viash/config/resources/RScript.scala index 1d59dda1c..02eef256b 100644 --- a/src/main/scala/io/viash/config/resources/RScript.scala +++ b/src/main/scala/io/viash/config/resources/RScript.scala @@ -27,7 +27,7 @@ import io.viash.config.Config @description("""An executable R script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. - |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin) + |When defined in test_resources, all entries will be executed during `viash test`.""") @subclass("r_script") case class RScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/config/resources/Resource.scala b/src/main/scala/io/viash/config/resources/Resource.scala index 1402c533f..2867f07c5 100644 --- a/src/main/scala/io/viash/config/resources/Resource.scala +++ b/src/main/scala/io/viash/config/resources/Resource.scala @@ -35,14 +35,14 @@ import io.viash.schemas._ | * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`. | * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`. | * is_executable: `true` / `false`, whether the resulting resource file should be made executable. - |""".stripMargin) + |""") @example( """resources: | - type: r_script | path: script.R | - type: file | path: resource1.txt - |""".stripMargin, + |""", "yaml") @subclass("BashScript") @subclass("CSharpScript") diff --git a/src/main/scala/io/viash/config/resources/ScalaScript.scala b/src/main/scala/io/viash/config/resources/ScalaScript.scala index 4bd7969f9..8a41e1949 100644 --- a/src/main/scala/io/viash/config/resources/ScalaScript.scala +++ b/src/main/scala/io/viash/config/resources/ScalaScript.scala @@ -27,7 +27,7 @@ import io.viash.config.Config @description("""An executable Scala script. |When defined in resources, only the first entry will be executed when running the built component or when running `viash run`. - |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin) + |When defined in test_resources, all entries will be executed during `viash test`.""") @subclass("scala_script") case class ScalaScript( path: Option[String] = None, diff --git a/src/main/scala/io/viash/engines/DockerEngine.scala b/src/main/scala/io/viash/engines/DockerEngine.scala index ba4fced3e..e4b6c181e 100644 --- a/src/main/scala/io/viash/engines/DockerEngine.scala +++ b/src/main/scala/io/viash/engines/DockerEngine.scala @@ -32,7 +32,7 @@ import io.viash.helpers.DockerImageInfo @description( """Run a Viash component on a Docker backend engine. |By specifying which dependencies your component needs, users will be able to build a docker container from scratch using the setup flag, or pull it from a docker repository. - |""".stripMargin) + |""") @example( """engines: | - type: docker @@ -40,7 +40,7 @@ import io.viash.helpers.DockerImageInfo | setup: | - type: apt | packages: [ curl ] - |""".stripMargin, + |""", "yaml") @subclass("docker") final case class DockerEngine( @@ -109,7 +109,7 @@ final case class DockerEngine( | - @[yum](yum_req) | |The order in which these dependencies are specified determines the order in which they will be installed. - |"""/*.stripMargin*/) + |""") @default("Empty") setup: List[Requirements] = Nil, diff --git a/src/main/scala/io/viash/engines/Engine.scala b/src/main/scala/io/viash/engines/Engine.scala index 6862bca58..7c45ad543 100644 --- a/src/main/scala/io/viash/engines/Engine.scala +++ b/src/main/scala/io/viash/engines/Engine.scala @@ -26,13 +26,13 @@ import io.viash.schemas._ | | * @[Docker](docker_engine) | * @[Native](native_engine) - |""".stripMargin) + |""") @example( """engines: | - type: docker | image: "bash:4.0" | - type: native - |""".stripMargin, + |""", "yaml") @subclass("DockerEngine") @subclass("NativeEngine") diff --git a/src/main/scala/io/viash/engines/NativeEngine.scala b/src/main/scala/io/viash/engines/NativeEngine.scala index 0c13ca4af..4525a1e0b 100644 --- a/src/main/scala/io/viash/engines/NativeEngine.scala +++ b/src/main/scala/io/viash/engines/NativeEngine.scala @@ -22,11 +22,11 @@ import io.viash.schemas._ @description( """Running a Viash component on a native engine means that the script will be executed in your current environment. |Any dependencies are assumed to have been installed by the user, so the native engine is meant for developers (who know what they're doing) or for simple bash scripts (which have no extra dependencies). - |""".stripMargin) + |""") @example( """engines: | - type: native - |""".stripMargin, + |""", "yaml") @subclass("native") final case class NativeEngine( diff --git a/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala b/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala index 1fa1543a8..2be4e031b 100644 --- a/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala @@ -25,7 +25,7 @@ import io.viash.schemas._ """setup: | - type: apk | packages: [ sl ] - |""".stripMargin, + |""", "yaml") @subclass("apk") case class ApkRequirements( diff --git a/src/main/scala/io/viash/engines/requirements/AptRequirements.scala b/src/main/scala/io/viash/engines/requirements/AptRequirements.scala index 80e81bcdb..830e2bc9e 100644 --- a/src/main/scala/io/viash/engines/requirements/AptRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/AptRequirements.scala @@ -25,7 +25,7 @@ import io.viash.schemas._ """setup: | - type: apt | packages: [ sl ] - |""".stripMargin, + |""", "yaml") @subclass("apt") case class AptRequirements( diff --git a/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala b/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala index 40d7e23a6..652f0d7eb 100644 --- a/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala @@ -23,11 +23,11 @@ import io.viash.schemas._ @description("Specify which Docker commands should be run during setup.") @example( """setup: - # - type: docker - # build_args: "R_VERSION=hello_world" - # run: | - # echo 'Run a custom command' - # echo 'Foo' > /path/to/file.txt""".stripMargin('#'), + | - type: docker + | build_args: "R_VERSION=hello_world" + | run: | + | echo 'Run a custom command' + | echo 'Foo' > /path/to/file.txt""", "yaml") @subclass("docker") case class DockerRequirements( @@ -48,8 +48,8 @@ case class DockerRequirements( @description("Specifies which `RUN` entries to add to the Dockerfile while building it.") @example("""run: | - # echo 'Run a custom command' - # echo 'Foo' > /path/to/file.txt""".stripMargin('#'), "yaml") + | echo 'Run a custom command' + | echo 'Foo' > /path/to/file.txt""", "yaml") @default("Empty") run: OneOrMore[String] = Nil, diff --git a/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala b/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala index 9bf3a97ef..67bcc29fd 100644 --- a/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala @@ -28,7 +28,7 @@ import io.viash.schemas._ | git: "https://some.git.repository/org/repo" | github: "owner/repository" | url: "https://github.com/org/repo/archive/HEAD.zip" - |""".stripMargin, + |""", "yaml") @subclass("javascript") case class JavaScriptRequirements( diff --git a/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala b/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala index be7f1c3d0..bd29bef7c 100644 --- a/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala @@ -27,7 +27,7 @@ import io.viash.schemas._ | pip: numpy | github: [ jkbr/httpie, foo/bar ] | url: "https://github.com/some_org/some_pkg/zipball/master" - |""".stripMargin, + |""", "yaml") @subclass("python") case class PythonRequirements( @@ -88,7 +88,7 @@ case class PythonRequirements( @description("Specifies a code block to run as part of the build.") @example("""script: | | print("Running custom code") - | x = 1 + 1 == 2"""/*.stripMargin*/, "yaml") + | x = 1 + 1 == 2""", "yaml") @default("Empty") script: OneOrMore[String] = Nil, diff --git a/src/main/scala/io/viash/engines/requirements/RRequirements.scala b/src/main/scala/io/viash/engines/requirements/RRequirements.scala index b1e31b3c4..02383db89 100644 --- a/src/main/scala/io/viash/engines/requirements/RRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/RRequirements.scala @@ -27,7 +27,7 @@ import io.viash.schemas._ | cran: anndata | bioc: [ AnnotationDbi, SingleCellExperiment ] | github: rcannood/SCORPIUS - |""".stripMargin, + |""", "yaml") @subclass("r") case class RRequirements( @@ -79,7 +79,7 @@ case class RRequirements( @description("Specifies a code block to run as part of the build.") @example("""script: | | cat("Running custom code\n") - | install.packages("anndata")"""/*.stripMargin*/, "yaml") + | install.packages("anndata")""", "yaml") @default("Empty") script: OneOrMore[String] = Nil, diff --git a/src/main/scala/io/viash/engines/requirements/Requirements.scala b/src/main/scala/io/viash/engines/requirements/Requirements.scala index b5c8be0eb..aff7dca1b 100644 --- a/src/main/scala/io/viash/engines/requirements/Requirements.scala +++ b/src/main/scala/io/viash/engines/requirements/Requirements.scala @@ -30,7 +30,7 @@ import io.viash.schemas._ | - @[R](r_req) | - @[Ruby](ruby_req) | - @[yum](yum_req) - |""".stripMargin) + |""") @subclass("ApkRequirements") @subclass("AptRequirements") @subclass("DockerRequirements") diff --git a/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala b/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala index fb70029bf..17340e2bc 100644 --- a/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala @@ -25,7 +25,7 @@ import io.viash.schemas._ """setup: | - type: ruby | packages: [ rspec ] - |""".stripMargin, + |""", "yaml") @subclass("ruby") case class RubyRequirements( diff --git a/src/main/scala/io/viash/engines/requirements/YumRequirements.scala b/src/main/scala/io/viash/engines/requirements/YumRequirements.scala index bd71c2c35..b0f696801 100644 --- a/src/main/scala/io/viash/engines/requirements/YumRequirements.scala +++ b/src/main/scala/io/viash/engines/requirements/YumRequirements.scala @@ -25,7 +25,7 @@ import io.viash.schemas._ """setup: | - type: yum | packages: [ sl ] - |""".stripMargin, + |""", "yaml") @subclass("yum") case class YumRequirements( diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala index 43c0ca993..49043729e 100644 --- a/src/main/scala/io/viash/functionality/Functionality.scala +++ b/src/main/scala/io/viash/functionality/Functionality.scala @@ -31,7 +31,7 @@ import scala.collection.immutable.ListMap @description( """The functionality-part of the config file describes the behaviour of the script in terms of arguments and resources. |By specifying a few restrictions (e.g. mandatory arguments) and adding some descriptions, Viash will automatically generate a stylish command-line interface for you. - |""".stripMargin) + |""") @deprecated("Functionality level is deprecated, all functionality fields are now located on the top level of the config file.", "0.9.0", "0.10.0") case class Functionality( @description("Name of the component and the filename of the executable when built with `viash build`.") @@ -48,20 +48,20 @@ case class Functionality( @description( """A list of @[authors](author). An author must at least have a name, but can also have a list of roles, an e-mail address, and a map of custom properties. - + - +Suggested values for roles are: - + - +| Role | Abbrev. | Description | - +|------|---------|-------------| - +| maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. | - +| author | aut | for persons who have made substantial contributions to the software. | - +| contributor | ctb| for persons who have made smaller contributions (such as code patches). - +| datacontributor | dtc | for persons or organisations that contributed data sets for the software - +| copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body. - +| funder | fnd | for persons or organizations that furnished financial support for the development of the software - + - +The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive. - +""".stripMargin('+')) + | + |Suggested values for roles are: + | + || Role | Abbrev. | Description | + ||------|---------|-------------| + || maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. | + || author | aut | for persons who have made substantial contributions to the software. | + || contributor | ctb| for persons who have made smaller contributions (such as code patches). + || datacontributor | dtc | for persons or organisations that contributed data sets for the software + || copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body. + || funder | fnd | for persons or organizations that furnished financial support for the development of the software + | + |The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive. + |""") @example( """authors: | - name: Jane Doe @@ -75,7 +75,7 @@ case class Functionality( | - name: Tim Farbe | roles: [author] | email: tim@far.be - |""".stripMargin, "yaml") + |""", "yaml") @since("Viash 0.3.1") @default("Empty") authors: List[Author] = Nil, @@ -87,7 +87,7 @@ case class Functionality( | - `description: Description of foo`, a description of the argument group. Multiline descriptions are supported. | - `arguments: [arg1, arg2, ...]`, list of the arguments. | - |""".stripMargin) + |""") @example( """argument_groups: | - name: "Input" @@ -107,7 +107,7 @@ case class Functionality( | - name: "--output_optional" | type: file | direction: output - |""".stripMargin, + |""", "yaml") @exampleWithDescription( """component_name @@ -125,7 +125,7 @@ case class Functionality( | | --optional_output | type: file - |""".stripMargin, + |""", "bash", "This results in the following output when calling the component with the `--help` argument:") @since("Viash 0.5.14") @@ -142,14 +142,14 @@ case class Functionality( | * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`. | * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`. | * is_executable: `true` / `false`, whether the resulting resource file should be made executable. - |""".stripMargin) + |""") @example( """resources: | - type: r_script | path: script.R | - type: file | path: resource1.txt - |""".stripMargin, + |""", "yaml") @default("Empty") resources: List[Resource] = Nil, @@ -157,9 +157,9 @@ case class Functionality( @description("A description of the component. This will be displayed with `--help`.") @example( """description: | - + This component performs function Y and Z. - + It is possible to make this a multiline string. - +""".stripMargin('+'), + | This component performs function Y and Z. + | It is possible to make this a multiline string. + |""", "yaml") description: Option[String] = None, @@ -175,7 +175,7 @@ case class Functionality( | - type: r_script | path: tests/test2.R | - path: resource1.txt - |""".stripMargin, + |""", "yaml") @default("Empty") test_resources: List[Resource] = Nil, @@ -184,7 +184,7 @@ case class Functionality( @example( """info: | twitter: wizzkid - | classes: [ one, two, three ]""".stripMargin, "yaml") + | classes: [ one, two, three ]""", "yaml") @since("Viash 0.4.0") @default("Empty") info: Json = Json.Null, @@ -198,12 +198,12 @@ case class Functionality( """@[Computational requirements](computational_requirements) related to running the component. |`cpus` specifies the maximum number of (logical) cpus a component is allowed to use., whereas |`memory` specifies the maximum amount of memory a component is allowed to allicate. Memory units must be - |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""".stripMargin) + |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""") @example( """requirements: | cpus: 5 | memory: 10GB - |""".stripMargin, + |""", "yaml") @since("Viash 0.6.0") @default("Empty") @@ -217,21 +217,21 @@ case class Functionality( | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |""", "yaml", "Full specification of a repository") @exampleWithDescription( """dependencies: | - name: qc/multiqc | repository: "github://openpipelines-bio/modules:0.3.0" - |""".stripMargin, + |""", "yaml", "Full specification of a repository using sugar syntax") @exampleWithDescription( """dependencies: | - name: qc/multiqc | repository: "openpipelines-bio" - |""".stripMargin, + |""", "yaml", "Reference to a repository fully specified under 'repositories'") @default("Empty") @@ -239,14 +239,14 @@ case class Functionality( @description( """(Pre-)defines @[repositories](repository) that can be used as repository in dependencies. - |Allows reusing repository definitions in case it is used in multiple dependencies.""".stripMargin) + |Allows reusing repository definitions in case it is used in multiple dependencies.""") @example( """repositories: | - name: openpipelines-bio | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |""", "yaml") @default("Empty") repositories: List[RepositoryWithName] = Nil, @@ -280,7 +280,7 @@ case class Functionality( | journal={Baz}, | year={2024} | } - |""".stripMargin, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") references: References = References(), @@ -293,7 +293,7 @@ case class Functionality( | homepage: "https://viash.io" | documentation: "https://viash.io/reference/" | issue_tracker: "https://github.com/viash-io/viash/issues" - |""".stripMargin, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") links: Links = Links(), @@ -311,7 +311,7 @@ case class Functionality( | - @[boolean](arg_boolean) | - @[boolean_true](arg_boolean_true) | - @[boolean_false](arg_boolean_false) - |""".stripMargin) + |""") @example( """arguments: | - name: --foo @@ -326,7 +326,7 @@ case class Functionality( | multiple_sep: ";" | - name: --bar | type: string - |""".stripMargin, + |""", "yaml") @default("Empty") arguments: List[Argument[_]] = Nil, diff --git a/src/main/scala/io/viash/helpers/SysEnv.scala b/src/main/scala/io/viash/helpers/SysEnv.scala index 6e30349c0..580f0bf61 100644 --- a/src/main/scala/io/viash/helpers/SysEnv.scala +++ b/src/main/scala/io/viash/helpers/SysEnv.scala @@ -28,14 +28,14 @@ trait SysEnvTrait { """If `VIASH_HOME` is not defined, the fallback `HOME`/.viash is used. | |Location where specific downloaded versions of Viash will be cached and run from. - |""".stripMargin) + |""") def viashHome: String @nameOverride("VIASH_VERSION") @description( """A specific Viash version can be set to run the commands with. If so required, the specific Viash version will be downloaded. |This is useful when replicating older results or building Viash components that use outdated code. - |""".stripMargin) + |""") @example( """VIASH_VERSION=0.7.0 viash ns build""", "sh" diff --git a/src/main/scala/io/viash/packageConfig/PackageConfig.scala b/src/main/scala/io/viash/packageConfig/PackageConfig.scala index e47fb86c6..46e75817b 100644 --- a/src/main/scala/io/viash/packageConfig/PackageConfig.scala +++ b/src/main/scala/io/viash/packageConfig/PackageConfig.scala @@ -42,7 +42,7 @@ import io.viash.config.{Author, Links, References} |config_mods: | | .runners[.type == 'nextflow'].directives.tag := '$id' | .runners[.type == 'nextflow'].config.script := 'includeConfig("configs/custom.config")' - |""".stripMargin, "yaml" + |""", "yaml" ) @since("Viash 0.6.4") case class PackageConfig( @@ -72,7 +72,7 @@ case class PackageConfig( @example( """description: | | A (multiline) description of the purpose of this package - | and the components it contains.""".stripMargin, "yaml") + | and the components it contains.""", "yaml") @since("Viash 0.9.0") description: Option[String] = None, @@ -80,7 +80,7 @@ case class PackageConfig( @example( """info: | twitter: wizzkid - | classes: [ one, two, three ]""".stripMargin, "yaml") + | classes: [ one, two, three ]""", "yaml") @default("Empty") @since("Viash 0.9.0") info: Json = Json.Null, @@ -92,7 +92,7 @@ case class PackageConfig( | type: github | uri: openpipelines-bio/modules | tag: 0.3.0 - |""".stripMargin, + |""", "yaml") @default("Empty") @since("Viash 0.9.0") @@ -137,7 +137,7 @@ case class PackageConfig( | - name: Tim Farbe | roles: [author] | email: tim@far.be - |""".stripMargin, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") authors: List[Author] = Nil, @@ -171,7 +171,7 @@ case class PackageConfig( | journal={Baz}, | year={2024} | } - |""".stripMargin, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") references: References = References(), @@ -184,7 +184,7 @@ case class PackageConfig( | homepage: "https://viash.io" | documentation: "https://viash.io/reference/" | issue_tracker: "https://github.com/viash-io/viash/issues" - |""".stripMargin, "yaml") + |""", "yaml") @default("Empty") @since("Viash 0.9.0") links: Links = Links(), diff --git a/src/main/scala/io/viash/platforms/DockerPlatform.scala b/src/main/scala/io/viash/platforms/DockerPlatform.scala index 7db7d5c3f..7bb600e66 100644 --- a/src/main/scala/io/viash/platforms/DockerPlatform.scala +++ b/src/main/scala/io/viash/platforms/DockerPlatform.scala @@ -27,7 +27,7 @@ import io.viash.engines.docker.{DockerResolveVolume, Automatic} @description( """Run a Viash component on a Docker backend platform. |By specifying which dependencies your component needs, users will be able to build a docker container from scratch using the setup flag, or pull it from a docker repository. - |""".stripMargin) + |""") @example( """platforms: | - type: docker @@ -35,7 +35,7 @@ import io.viash.engines.docker.{DockerResolveVolume, Automatic} | setup: | - type: apt | packages: [ curl ] - |""".stripMargin, + |""", "yaml") @deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0") @subclass("docker") @@ -96,7 +96,7 @@ case class DockerPlatform( """port: | - 80 | - 8080 - |""".stripMargin, + |""", "yaml") @default("Empty") port: OneOrMore[String] = Nil, @@ -107,24 +107,24 @@ case class DockerPlatform( @description( """The Docker setup strategy to use when building a container. - + - +| Strategy | Description | - +|-----|----------| - +| `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy. - +| `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled. - +| `ifneedbebuild` | Build the image if it does not exist locally. - +| `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled. - +| `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). - +| `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it does not exist. - +| `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it does not exist. - +| `ifneedbepull` | If the image does not exist locally, pull the image. - +| `ifneedbepullelsebuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image from scratch. - +| `ifneedbepullelsecachedbuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image with caching enabled. - +| `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). - +| `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet. - +| `donothing` / `meh` | Do not build or pull anything. - + - +""".stripMargin('+')) + | + || Strategy | Description | + ||-----|----------| + || `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy. + || `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled. + || `ifneedbebuild` | Build the image if it does not exist locally. + || `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled. + || `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). + || `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it does not exist. + || `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it does not exist. + || `ifneedbepull` | If the image does not exist locally, pull the image. + || `ifneedbepullelsebuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image from scratch. + || `ifneedbepullelsecachedbuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image with caching enabled. + || `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). + || `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet. + || `donothing` / `meh` | Do not build or pull anything. + | + |""") @example("setup_strategy: alwaysbuild", "yaml") @default("ifneedbepullelsecachedbuild") setup_strategy: DockerSetupStrategy = IfNeedBePullElseCachedBuild, @@ -152,7 +152,7 @@ case class DockerPlatform( | - @[yum](yum_req) | |The order in which these dependencies are specified determines the order in which they will be installed. - |""".stripMargin) + |""") @default("Empty") setup: List[Requirements] = Nil, diff --git a/src/main/scala/io/viash/platforms/NativePlatform.scala b/src/main/scala/io/viash/platforms/NativePlatform.scala index e01a3256d..fb60d6cd2 100644 --- a/src/main/scala/io/viash/platforms/NativePlatform.scala +++ b/src/main/scala/io/viash/platforms/NativePlatform.scala @@ -22,11 +22,11 @@ import io.viash.schemas._ @description( """Running a Viash component on a native platform means that the script will be executed in your current environment. |Any dependencies are assumed to have been installed by the user, so the native platform is meant for developers (who know what they're doing) or for simple bash scripts (which have no extra dependencies). - |""".stripMargin) + |""") @example( """platforms: | - type: native - |""".stripMargin, + |""", "yaml") @deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0") @subclass("native") diff --git a/src/main/scala/io/viash/platforms/NextflowPlatform.scala b/src/main/scala/io/viash/platforms/NextflowPlatform.scala index 1de8ba6cc..bf89c470f 100644 --- a/src/main/scala/io/viash/platforms/NextflowPlatform.scala +++ b/src/main/scala/io/viash/platforms/NextflowPlatform.scala @@ -23,14 +23,14 @@ import io.viash.runners.nextflow._ /** * A Platform class for generating Nextflow (DSL2) modules. */ -@description("""Platform for generating Nextflow VDSL3 modules.""".stripMargin) +@description("""Platform for generating Nextflow VDSL3 modules.""") // todo: add link to guide @example( """platforms: | - type: nextflow | directives: | label: [lowcpu, midmem] - |""".stripMargin, + |""", "yaml") @deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0") @subclass("nextflow") @@ -45,13 +45,13 @@ case class NextflowPlatform( // nxf params @description( """@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts. - |"""/*.stripMargin*/) + |""") @example( """directives: | container: rocker/r-ver:4.1 | label: highcpu | cpus: 4 - | memory: 16 GB"""/*.stripMargin*/, + | memory: 16 GB""", "yaml") @default("Empty") directives: NextflowDirectives = NextflowDirectives(), @@ -66,17 +66,17 @@ case class NextflowPlatform( || `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` | || `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. If equal to `"state"`, also a `.state.yaml` file will be published in the publish dir. Will throw an error if `params.publishDir` is not defined. | `false` | | - |"""/*.stripMargin*/) + |""") @example( """auto: - | publish: true"""/*.stripMargin*/, + | publish: true""", "yaml") @default( """simplifyInput: true |simplifyOutput: false |transcript: false |publish: false - |"""/*.stripMargin*/) + |""") auto: NextflowAuto = NextflowAuto(), @description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.") diff --git a/src/main/scala/io/viash/platforms/Platform.scala b/src/main/scala/io/viash/platforms/Platform.scala index 196a96d4f..351e0f0ae 100644 --- a/src/main/scala/io/viash/platforms/Platform.scala +++ b/src/main/scala/io/viash/platforms/Platform.scala @@ -26,7 +26,7 @@ import io.viash.engines.requirements.Requirements | * @[Native](platform_native) | * @[Docker](platform_docker) | * @[Nextflow](platform_nextflow) - |""".stripMargin) + |""") @example( """platforms: | - type: docker @@ -35,7 +35,7 @@ import io.viash.engines.requirements.Requirements | - type: nextflow | directives: | label: [lowcpu, midmem] - |""".stripMargin, + |""", "yaml") @deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0") @subclass("NativePlatform") diff --git a/src/main/scala/io/viash/runners/ExecutableRunner.scala b/src/main/scala/io/viash/runners/ExecutableRunner.scala index 99f1fa3b8..d72df7907 100644 --- a/src/main/scala/io/viash/runners/ExecutableRunner.scala +++ b/src/main/scala/io/viash/runners/ExecutableRunner.scala @@ -45,12 +45,12 @@ import io.viash.schemas._ |This runner is also used for the @[native](native_engine) engine. | |This runner is also used for the @[docker](docker_engine) engine. - |""".stripMargin) + |""") @example( """runners: | - type: executable | port: 8080 - |""".stripMargin, + |""", "yaml") @subclass("executable") final case class ExecutableRunner( @@ -64,7 +64,7 @@ final case class ExecutableRunner( """port: | - 80 | - 8080 - |""".stripMargin, + |""", "yaml") @default("Empty") port: OneOrMore[String] = Nil, @@ -75,24 +75,24 @@ final case class ExecutableRunner( @description( """The Docker setup strategy to use when building a docker engine enrivonment. - + - +| Strategy | Description | - +|-----|----------| - +| `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy. - +| `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled. - +| `ifneedbebuild` | Build the image if it does not exist locally. - +| `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled. - +| `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). - +| `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it doesn't exist. - +| `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it doesn't exist. - +| `ifneedbepull` | If the image does not exist locally, pull the image. - +| `ifneedbepullelsebuild` | If the image does not exist locally, pull the image. If the image does exist, build it. - +| `ifneedbepullelsecachedbuild` | If the image does not exist locally, pull the image. If the image does exist, build it with caching enabled. - +| `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). - +| `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet. - +| `donothing` / `meh` | Do not build or pull anything. - + - +""".stripMargin('+')) + | + || Strategy | Description | + ||-----|----------| + || `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy. + || `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled. + || `ifneedbebuild` | Build the image if it does not exist locally. + || `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled. + || `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). + || `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it doesn't exist. + || `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it doesn't exist. + || `ifneedbepull` | If the image does not exist locally, pull the image. + || `ifneedbepullelsebuild` | If the image does not exist locally, pull the image. If the image does exist, build it. + || `ifneedbepullelsecachedbuild` | If the image does not exist locally, pull the image. If the image does exist, build it with caching enabled. + || `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry). + || `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet. + || `donothing` / `meh` | Do not build or pull anything. + | + |""") @example("setup_strategy: alwaysbuild", "yaml") @default("ifneedbepullelsecachedbuild") docker_setup_strategy: DockerSetupStrategy = IfNeedBePullElseCachedBuild, diff --git a/src/main/scala/io/viash/runners/NextflowRunner.scala b/src/main/scala/io/viash/runners/NextflowRunner.scala index 37e617c48..81c68d3e7 100644 --- a/src/main/scala/io/viash/runners/NextflowRunner.scala +++ b/src/main/scala/io/viash/runners/NextflowRunner.scala @@ -33,13 +33,13 @@ import io.viash.config.resources.{Executable, NextflowScript, PlainFile} @description( """Run a Viash component on a Nextflow backend engine. - |""".stripMargin) + |""") @example( """runners: | - type: nextflow | directives: | label: [lowcpu, midmem] - |""".stripMargin, + |""", "yaml") @subclass("nextflow") final case class NextflowRunner( @@ -53,13 +53,13 @@ final case class NextflowRunner( // nxf params @description( """@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts. - |"""/*.stripMargin*/) + |""") @example( """directives: | container: rocker/r-ver:4.1 | label: highcpu | cpus: 4 - | memory: 16 GB"""/*.stripMargin*/, + | memory: 16 GB""", "yaml") @default("Empty") directives: NextflowDirectives = NextflowDirectives(), @@ -74,17 +74,17 @@ final case class NextflowRunner( || `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` | || `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. If equal to `"state"`, also a `.state.yaml` file will be published in the publish dir. Will throw an error if `params.publishDir` is not defined. | `false` | | - |"""/*.stripMargin*/) + |""") @example( """auto: - | publish: true"""/*.stripMargin*/, + | publish: true""", "yaml") @default( """simplifyInput: true |simplifyOutput: false |transcript: false |publish: false - |"""/*.stripMargin*/) + |""") auto: NextflowAuto = NextflowAuto(), @description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.") diff --git a/src/main/scala/io/viash/runners/Runner.scala b/src/main/scala/io/viash/runners/Runner.scala index dc40ec6f8..5af6563e5 100644 --- a/src/main/scala/io/viash/runners/Runner.scala +++ b/src/main/scala/io/viash/runners/Runner.scala @@ -27,12 +27,12 @@ import io.viash.config.Config | | * @[Executable](executable_runner) | * @[Nextflow](nextflow_runner) - |""".stripMargin) + |""") @example( """runners: | - type: executable | - type: nextflow - |""".stripMargin, + |""", "yaml") @subclass("ExecutableRunner") @subclass("NextflowRunner") diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala b/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala index 6451b3d59..faaf2667f 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala @@ -25,7 +25,7 @@ case class NextflowAuto( """If `true`, an input tuple only containing only a single File (e.g. `["foo", file("in.h5ad")]`) is automatically transformed to a map (i.e. `["foo", [ input: file("in.h5ad") ] ]`). | |Default: `true`. - |"""/*.stripMargin*/) + |""") @default("True") simplifyInput: Boolean = true, @@ -33,7 +33,7 @@ case class NextflowAuto( """If `true`, an output tuple containing a map with a File (e.g. `["foo", [ output: file("out.h5ad") ] ]`) is automatically transformed to a map (i.e. `["foo", file("out.h5ad")]`). | |Default: `false`. - |"""/*.stripMargin*/) + |""") @default("False") simplifyOutput: Boolean = false, @@ -43,7 +43,7 @@ case class NextflowAuto( |Will throw an error if neither are defined. | |Default: `false`. - |"""/*.stripMargin*/) + |""") @default("False") transcript: Boolean = false, @@ -53,7 +53,7 @@ case class NextflowAuto( |Will throw an error if `params.publishDir` is not defined. | |Default: `false`. - |"""/*.stripMargin*/) + |""") @default("False") publish: Either[Boolean, String] = Left(false) ) { diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala index 7018b27e6..40ea310eb 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala @@ -33,7 +33,7 @@ case class NextflowConfig( | |Conceptually it is possible for a Viash Config to overwrite the full labels parameter, however likely it is more efficient to add additional labels |in the Viash Package with a config mod. - |"""/*.stripMargin*/) + |""") @exampleWithDescription( """labels: | lowmem: "memory = 4.GB" @@ -44,7 +44,7 @@ case class NextflowConfig( | highcpu: "cpus = 20" | vhighmem: "memory = 100.GB" | vhighcpu: "cpus = 40" - |"""/*.stripMargin*/, + |""", "yaml", "Replace the default labels with a different set of labels") @exampleWithDescription( @@ -55,14 +55,14 @@ case class NextflowConfig( """config_mods: | | .runners[.type == "nextflow"].config.labels.lowmem := "memory = 4.GB" | .runners[.type == "nextflow"].config.labels.lowcpu := "cpus = 4" - |"""/*.stripMargin*/, + |""", "viash_package_file", "Add 'lowmem' and 'lowcpu' to the default labels by using the Viash Package file" ) @exampleWithDescription( """config_mods: | | .runners[.type == "nextflow"].config.labels := { lowmem: "memory = 4.GB", lowcpu: "cpus = 4", midmem: "memory = 25.GB", midcpu: "cpus = 10", highmem: "memory = 50.GB", highcpu: "cpus = 20", vhighmem: "memory = 100.GB", vhighcpu: "cpus = 40" } - |"""/*.stripMargin*/, + |""", "viash_package_file", "Replace the default labels with a different set of labels by using the Viash Package file" ) @@ -90,14 +90,14 @@ case class NextflowConfig( @description( """Includes a single string or list of strings into the nextflow.config file. |This can be used to add custom profiles or include an additional config file. - |"""/*.stripMargin*/) + |""") @example( """script: | - | | profiles { | ... | } - |"""/*.stripMargin*/, + |""", "yaml") @example("""script: includeConfig("config.config")""", "yaml") @default("Empty") diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala b/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala index 10a033784..187b5e667 100644 --- a/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala +++ b/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala @@ -23,13 +23,13 @@ import io.viash.schemas._ // todo: assert contents? @description( """Directives are optional settings that affect the execution of the process. - |""".stripMargin) + |""") @example( """directives: | container: rocker/r-ver:4.1 | label: highcpu | cpus: 4 - | memory: 16 GB""".stripMargin, + | memory: 16 GB""", "yaml") case class NextflowDirectives( @description( @@ -38,7 +38,7 @@ case class NextflowDirectives( |Viash implements this directive as a map with accepted keywords: `type`, `limit`, `request`, and `runtime`. | |See [`accelerator`](https://www.nextflow.io/docs/latest/process.html#accelerator). - |""".stripMargin) + |""") @example("""[ limit: 4, type: "nvidia-tesla-k80" ]""", "yaml") @default("Empty") accelerator: Map[String, String] = Map(), @@ -47,7 +47,7 @@ case class NextflowDirectives( """The `afterScript` directive allows you to execute a custom (Bash) snippet immediately after the main process has run. This may be useful to clean up your staging area. | |See [`afterScript`](https://www.nextflow.io/docs/latest/process.html#afterscript). - |""".stripMargin) + |""") @example("""source /cluster/bin/cleanup""", "yaml") afterScript: Option[String] = None, @@ -55,7 +55,7 @@ case class NextflowDirectives( """The `beforeScript` directive allows you to execute a custom (Bash) snippet before the main process script is run. This may be useful to initialise the underlying cluster environment or for other custom initialisation. | |See [`beforeScript`](https://www.nextflow.io/docs/latest/process.html#beforeScript). - |""".stripMargin) + |""") @example("""source /cluster/bin/setup""", "yaml") beforeScript: Option[String] = None, @@ -69,7 +69,7 @@ case class NextflowDirectives( |Accepted values are: `true`, `false`, `"deep"`, and `"lenient"`. | |See [`cache`](https://www.nextflow.io/docs/latest/process.html#cache). - |""".stripMargin) + |""") @example("true", "yaml") @example("false", "yaml") @example(""""deep"""", "yaml") @@ -82,7 +82,7 @@ case class NextflowDirectives( |Nextflow automatically sets up an environment for the given package names listed by in the `conda` directive. | |See [`conda`](https://www.nextflow.io/docs/latest/process.html#conda). - |""".stripMargin) + |""") @example(""""bwa=0.7.15"""", "yaml") @example(""""bwa=0.7.15 fastqc=0.11.5"""", "yaml") @example("""["bwa=0.7.15", "fastqc=0.11.5"]""", "yaml") @@ -97,7 +97,7 @@ case class NextflowDirectives( |Viash implements allows either a string value or a map. In case a map is used, the allowed keys are: `registry`, `image`, and `tag`. The `image` value must be specified. | |See [`container`](https://www.nextflow.io/docs/latest/process.html#container). - |""".stripMargin) + |""") @example(""""foo/bar:tag"""", "yaml") @exampleWithDescription("""[ registry: "reg", image: "im", tag: "ta" ]""", "yaml", """This is transformed to `"reg/im:ta"`:""") @exampleWithDescription("""[ image: "im" ]""", "yaml", """This is transformed to `"im:latest"`:""") @@ -107,7 +107,7 @@ case class NextflowDirectives( """The `containerOptions` directive allows you to specify any container execution option supported by the underlying container engine (ie. Docker, Singularity, etc). This can be useful to provide container settings only for a specific process e.g. mount a custom path. | |See [`containerOptions`](https://www.nextflow.io/docs/latest/process.html#containeroptions). - |""".stripMargin) + |""") @example(""""--foo bar"""", "yaml") @example("""["--foo bar", "-f b"]""", "yaml") @default("Empty") @@ -117,7 +117,7 @@ case class NextflowDirectives( """The `cpus` directive allows you to define the number of (logical) CPU required by the process' task. | |See [`cpus`](https://www.nextflow.io/docs/latest/process.html#cpus). - |""".stripMargin) + |""") @example("1", "yaml") @example("10", "yaml") cpus: Option[Either[Int, String]] = None, @@ -126,7 +126,7 @@ case class NextflowDirectives( """The `disk` directive allows you to define how much local disk storage the process is allowed to use. | |See [`disk`](https://www.nextflow.io/docs/latest/process.html#disk). - |""".stripMargin) + |""") @example(""""1 GB"""", "yaml") @example(""""2TB"""", "yaml") @example(""""3.2KB"""", "yaml") @@ -137,7 +137,7 @@ case class NextflowDirectives( """By default the stdout produced by the commands executed in all processes is ignored. By setting the `echo` directive to true, you can forward the process stdout to the current top running process stdout file, showing it in the shell terminal. | |See [`echo`](https://www.nextflow.io/docs/latest/process.html#echo). - |""".stripMargin) + |""") @example("true", "yaml") @example("false", "yaml") echo: Option[Either[Boolean, String]] = None, @@ -154,7 +154,7 @@ case class NextflowDirectives( || `retry` | Re-submit for execution a process returning an error condition. | | |See [`errorStrategy`](https://www.nextflow.io/docs/latest/process.html#errorstrategy). - |""".stripMargin) + |""") @example(""""terminate"""", "yaml") @example(""""finish"""", "yaml") errorStrategy: Option[String] = None, @@ -185,7 +185,7 @@ case class NextflowDirectives( || uge | Alias for the sge executor. | | |See [`executor`](https://www.nextflow.io/docs/latest/process.html#executor). - |""".stripMargin) + |""") @example(""""local"""", "yaml") @example(""""sge"""", "yaml") executor: Option[String] = None, @@ -194,7 +194,7 @@ case class NextflowDirectives( """The `label` directive allows the annotation of processes with mnemonic identifier of your choice. | |See [`label`](https://www.nextflow.io/docs/latest/process.html#label). - |""".stripMargin) + |""") @example(""""big_mem"""", "yaml") @example(""""big_cpu"""", "yaml") @example("""["big_mem", "big_cpu"]""", "yaml") @@ -205,7 +205,7 @@ case class NextflowDirectives( """ The `machineType` can be used to specify a predefined Google Compute Platform machine type when running using the Google Life Sciences executor. | |See [`machineType`](https://www.nextflow.io/docs/latest/process.html#machinetype). - |""".stripMargin) + |""") @example(""""n1-highmem-8"""", "yaml") machineType: Option[String] = None, @@ -213,7 +213,7 @@ case class NextflowDirectives( """The `maxErrors` directive allows you to specify the maximum number of times a process can fail when using the `retry` error strategy. By default this directive is disabled. | |See [`maxErrors`](https://www.nextflow.io/docs/latest/process.html#maxerrors). - |""".stripMargin) + |""") @example("1", "yaml") @example("3", "yaml") maxErrors: Option[Either[String, Int]] = None, @@ -224,7 +224,7 @@ case class NextflowDirectives( |If you want to execute a process in a sequential manner, set this directive to one. | |See [`maxForks`](https://www.nextflow.io/docs/latest/process.html#maxforks). - |""".stripMargin) + |""") @example("1", "yaml") @example("3", "yaml") maxForks: Option[Either[String, Int]] = None, @@ -233,7 +233,7 @@ case class NextflowDirectives( """The `maxRetries` directive allows you to define the maximum number of times a process instance can be re-submitted in case of failure. This value is applied only when using the retry error strategy. By default only one retry is allowed. | |See [`maxRetries`](https://www.nextflow.io/docs/latest/process.html#maxretries). - |""".stripMargin) + |""") @example("1", "yaml") @example("3", "yaml") maxRetries: Option[Either[String, Int]] = None, @@ -242,7 +242,7 @@ case class NextflowDirectives( """The `memory` directive allows you to define how much memory the process is allowed to use. | |See [`memory`](https://www.nextflow.io/docs/latest/process.html#memory). - |""".stripMargin) + |""") @example(""""1 GB"""", "yaml") @example(""""2TB"""", "yaml") @example(""""3.2KB"""", "yaml") @@ -257,7 +257,7 @@ case class NextflowDirectives( |In a process definition you can use the `module` directive to load a specific module version to be used in the process execution environment. | |See [`module`](https://www.nextflow.io/docs/latest/process.html#module). - |""".stripMargin) + |""") @example(""""ncbi-blast/2.2.27"""", "yaml") @example(""""ncbi-blast/2.2.27:t_coffee/10.0"""", "yaml") @example("""["ncbi-blast/2.2.27", "t_coffee/10.0"]""", "yaml") @@ -268,7 +268,7 @@ case class NextflowDirectives( """The `penv` directive allows you to define the parallel environment to be used when submitting a parallel task to the SGE resource manager. | |See [`penv`](https://www.nextflow.io/docs/latest/process.html#penv). - |""".stripMargin) + |""") @example(""""smp"""", "yaml") penv: Option[String] = None, @@ -276,7 +276,7 @@ case class NextflowDirectives( """The `pod` directive allows the definition of pods specific settings, such as environment variables, secrets and config maps when using the Kubernetes executor. | |See [`pod`](https://www.nextflow.io/docs/latest/process.html#pod). - |""".stripMargin) + |""") @example("""[ label: "key", value: "val" ]""", "yaml") @example("""[ annotation: "key", value: "val" ]""", "yaml") @example("""[ env: "key", value: "val" ]""", "yaml") @@ -291,7 +291,7 @@ case class NextflowDirectives( |The allowed values for `mode` are: `symlink`, `rellink`, `link`, `copy`, `copyNoFollow`, `move`. | |See [`publishDir`](https://www.nextflow.io/docs/latest/process.html#publishdir). - |""".stripMargin) + |""") @example("[]", "yaml") @example("""[ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ]""", "yaml") @exampleWithDescription(""""/path/to/dir"""", "yaml", """This is transformed to `[[ path: "/path/to/dir" ]]`:""") @@ -303,7 +303,7 @@ case class NextflowDirectives( """The `queue` directory allows you to set the queue where jobs are scheduled when using a grid based executor in your pipeline. | |See [`queue`](https://www.nextflow.io/docs/latest/process.html#queue). - |""".stripMargin) + |""") @example(""""long"""", "yaml") @example(""""short,long"""", "yaml") @example("""["short", "long"]""", "yaml") @@ -314,7 +314,7 @@ case class NextflowDirectives( """The `scratch` directive allows you to execute the process in a temporary folder that is local to the execution node. | |See [`scratch`](https://www.nextflow.io/docs/latest/process.html#scratch). - |""".stripMargin) + |""") @example("true", "yaml") @example(""""/path/to/scratch"""", "yaml") @example("""'$MY_PATH_TO_SCRATCH'""", "yaml") @@ -325,7 +325,7 @@ case class NextflowDirectives( """The `storeDir` directive allows you to define a directory that is used as a permanent cache for your process results. | |See [`storeDir`](https://www.nextflow.io/docs/latest/process.html#storeDir). - |""".stripMargin) + |""") @example(""""/path/to/storeDir"""", "yaml") storeDir: Option[String] = None, @@ -340,7 +340,7 @@ case class NextflowDirectives( || rellink | Input files are staged in the process work directory by creating a symbolic link with a relative path for each of them. | | |See [`stageInMode`](https://www.nextflow.io/docs/latest/process.html#stageinmode). - |""".stripMargin) + |""") @example(""""copy"""", "yaml") @example(""""link"""", "yaml") stageInMode: Option[String] = None, @@ -355,7 +355,7 @@ case class NextflowDirectives( || rsync | Output files are copied from the scratch directory to the work directory by using the rsync utility. | | |See [`stageOutMode`](https://www.nextflow.io/docs/latest/process.html#stageoutmode). - |""".stripMargin) + |""") @example(""""copy"""", "yaml") @example(""""link"""", "yaml") stageOutMode: Option[String] = None, @@ -366,7 +366,7 @@ case class NextflowDirectives( |For ease of use, the default tag is set to `"$id"`, which allows tracking the progression of the channel events through the workflow more easily. | |See [`tag`](https://www.nextflow.io/docs/latest/process.html#tag). - |""".stripMargin) + |""") @example(""""foo"""", "yaml") @default("""'$id'""") tag: Option[String] = Some("$id"), @@ -375,7 +375,7 @@ case class NextflowDirectives( """The `time` directive allows you to define how long a process is allowed to run. | |See [`time`](https://www.nextflow.io/docs/latest/process.html#time). - |""".stripMargin) + |""") @example(""""1h"""", "yaml") @example(""""2days"""", "yaml") @example(""""1day 6hours 3minutes 30seconds"""", "yaml") From e3cdc9bf91ab6f213f230f57cee273f789202cef Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 14:21:38 +0100 Subject: [PATCH 36/46] cleanup mirroring code --- .../scala/io/viash/helpers/Mirroring.scala | 91 +------------------ 1 file changed, 4 insertions(+), 87 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 9bd69cccb..01f951bd1 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -136,51 +136,9 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] import quotes.reflect.* val tpe = TypeRepr.of[T].typeSymbol - - def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = { - s.replaceAll("\\\\n", "\n").stripMargin(marginChar) - } - - def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { - l.map(i => i match { - // case Literal(Constant(value: String)) => - // unfinishedStringStripMargin(value, marginChar) - case Literal(value) => - unfinishedStringStripMargin(value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\""), marginChar) - case _ => - "unmatched in mapTreeList: " + i.toString() - }).mkString - } - // Traverse tree information and extract values or lists of values - def annotationToStrings(ann: Term): List[String] = { - // val name = ann.tree.tpe.toString() - val values = ann match { - case Apply(c, args: List[Tree]) => - args.collect({ - case i: Tree => - i match { - // Here 'Apply' contains lists - // While 'Select' has a single element - // case Literal(Constant(value: String)) => - // value - case Literal(value) => - value.value.toString().stripMargin - // value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") - // case Select(Select(a, b), stripMargin) => - // unfinishedStringStripMargin(b) - case Select(Apply(a, a2), b) if b.toString == "stripMargin" => - mapTreeList(a2) - case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => - val stripper = stripMargin.head.toString.charAt(1) - mapTreeList(a2, stripper) - case _ => - "unmatched in annotationToStrings: " + i.toString() - } - }) - } - values - } + def annotationToStrings(ann: Term): List[String] = + ann match {case Apply(_, args) => args.collect{ case Literal(constant) => constant.value.toString.stripMargin }} // We're not adding annotations of base classes here. // The base classes should be documented as well and the annotations will clash with the annotations of the specific class. @@ -200,50 +158,9 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, String, List[(String, List[String])])]] = { import quotes.reflect.* - def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = { - s.replaceAll("\\\\n", "\n").stripMargin(marginChar) - } - - def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = { - l.map(i => i match { - // case Literal(Constant(value: String)) => - // unfinishedStringStripMargin(value, marginChar) - case Literal(value) => - unfinishedStringStripMargin(value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\""), marginChar) - case _ => - "unmatched in mapTreeList: " + i.toString() - }).mkString - } - // Traverse tree information and extract values or lists of values - def annotationToStrings(ann: Term): List[String] = { - // val name = ann.tree.tpe.toString() - val values = ann match { - case Apply(c, args: List[Tree]) => - args.collect({ - case i: Tree => - i match { - // Here 'Apply' contains lists - // While 'Select' has a single element - // case Literal(Constant(value: String)) => - // value - case Literal(value) => - value.value.toString().stripMargin - // value.show(using Printer.ConstantCode).stripPrefix("\"").stripSuffix("\"") - // case Select(Select(a, b), stripMargin) => - // unfinishedStringStripMargin(b) - case Select(Apply(a, a2), b) if b.toString == "stripMargin" => - mapTreeList(a2) - case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" => - val stripper = stripMargin.head.toString.charAt(1) - mapTreeList(a2, stripper) - case _ => - "unmatched in annotationToStrings: " + i.toString() - } - }) - } - values - } + def annotationToStrings(ann: Term): List[String] = + ann match {case Apply(_, args) => args.collect{ case Literal(constant) => constant.value.toString.stripMargin }} // Use pattern matching to extract a simplified name def simpleName(tpe: TypeRepr): String = tpe match { From b2dc774b5f8833a024f0499bca8f9264fbe1defc Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 14:54:59 +0100 Subject: [PATCH 37/46] prune some stale code move some memberData to part2 because gha is failing for java 17. See what this does. \o/ --- src/main/scala/io/viash/helpers/Mirroring.scala | 8 -------- src/main/scala/io/viash/schemas/CollectedSchemas.scala | 10 +++++----- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala index 01f951bd1..002b3a528 100644 --- a/src/main/scala/io/viash/helpers/Mirroring.scala +++ b/src/main/scala/io/viash/helpers/Mirroring.scala @@ -29,7 +29,6 @@ inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctiona inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] } inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] } inline def annotationsOf[T]: List[(String, List[String])] = ${ annotationsOfImpl[T] } -// inline def membersOf[T]: List[String] = ${ membersOfImpl[T] } inline def memberTypeAnnotationsOf[T]: List[(String, String, List[(String, List[String])])] = ${ memberTypeAnnotationsOfImpl[T] } inline def historyOf[T]: List[String] = ${ historyOfImpl[T] } @@ -148,13 +147,6 @@ def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] Expr(annots) -// def membersOfImpl[T: Type](using Quotes): Expr[List[String]] = { -// import quotes.reflect.* -// val tpe = TypeRepr.of[T].typeSymbol -// val memberSymbols = tpe.caseFields.map(_.name) -// Expr(memberSymbols) -// } - def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, String, List[(String, List[String])])]] = { import quotes.reflect.* diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index aa4aa6ad4..d6f7fdc17 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -100,7 +100,10 @@ object CollectedSchemas { getMembers[RRequirements](), getMembers[RubyRequirements](), getMembers[YumRequirements](), - + ) + } + object memberData_part2 { + val part = List( getMembers[Argument[_]](), getMembers[BooleanArgument](), getMembers[BooleanTrueArgument](), @@ -110,10 +113,7 @@ object CollectedSchemas { getMembers[IntegerArgument](), getMembers[LongArgument](), getMembers[StringArgument](), - ) - } - object memberData_part2 { - val part = List( + getMembers[Resource](), getMembers[BashScript](), getMembers[CSharpScript](), From 88c8ef26b295575d7643d51fbd3de4b41ac96215 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 15:07:11 +0100 Subject: [PATCH 38/46] try splitting into 3 parts --- .../io/viash/schemas/CollectedSchemas.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index d6f7fdc17..fb2475461 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -90,7 +90,10 @@ object CollectedSchemas { getMembers[NativePlatform](), getMembers[DockerPlatform](), getMembers[NextflowPlatform](), - + ) + } + object memberData_part2 { + val part = List( getMembers[Requirements](), getMembers[ApkRequirements](), getMembers[AptRequirements](), @@ -100,10 +103,7 @@ object CollectedSchemas { getMembers[RRequirements](), getMembers[RubyRequirements](), getMembers[YumRequirements](), - ) - } - object memberData_part2 { - val part = List( + getMembers[Argument[_]](), getMembers[BooleanArgument](), getMembers[BooleanTrueArgument](), @@ -113,7 +113,10 @@ object CollectedSchemas { getMembers[IntegerArgument](), getMembers[LongArgument](), getMembers[StringArgument](), - + ) + } + object memberData_part3 { + val part = List( getMembers[Resource](), getMembers[BashScript](), getMembers[CSharpScript](), @@ -153,7 +156,7 @@ object CollectedSchemas { .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") } - val fullData = memberData_part1.part ++ memberData_part2.part + val fullData = memberData_part1.part ++ memberData_part2.part ++ memberData_part3.part // Main call for documentation output lazy val data: List[List[ParameterSchema]] = fullData.map(_.filter(p => !p.hasUndocumented && !p.hasInternalFunctionality)) From 7f4d5febb27ce1a112aba6473299cbeab9f4c247 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Fri, 22 Nov 2024 19:31:46 +0100 Subject: [PATCH 39/46] Cleanup of changed code, removing some code put in comments etc --- build.sbt | 1 - src/main/scala/io/viash/config/dependencies/package.scala | 2 -- src/main/scala/io/viash/engines/docker/package.scala | 1 - src/main/scala/io/viash/engines/package.scala | 1 - src/main/scala/io/viash/engines/requirements/package.scala | 1 - src/main/scala/io/viash/schemas/CollectedSchemas.scala | 2 -- src/main/scala/io/viash/schemas/ParameterSchema.scala | 1 - 7 files changed, 9 deletions(-) diff --git a/build.sbt b/build.sbt index 58c1f930e..86546ad55 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,6 @@ libraryDependencies ++= Seq( "org.scalactic" %% "scalactic" % "3.2.15" % "test", "org.scalatest" %% "scalatest" % "3.2.15" % "test", "org.rogach" %% "scallop" % "5.0.0", - // "org.scala-lang" % "scala-reflect" % scalaVersion.value, "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1", "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4", "dev.optics" %% "monocle-core" % "3.1.0", diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala index 7b880fb29..d260a53c5 100644 --- a/src/main/scala/io/viash/config/dependencies/package.scala +++ b/src/main/scala/io/viash/config/dependencies/package.scala @@ -26,7 +26,6 @@ package object dependencies { import io.viash.helpers.circe._ // encoders and decoders for Argument - // implicit val encodeEitherStringRepository: Encoder[Either[String, Repository]] = deriveConfiguredEncoderStrict implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict implicit val encodeGitRepository: Encoder.AsObject[GitRepository] = deriveConfiguredEncoderStrict implicit val encodeGithubRepository: Encoder.AsObject[GithubRepository] = deriveConfiguredEncoderStrict @@ -65,7 +64,6 @@ package object dependencies { objJson deepMerge typeJson } - // implicit val decodeEitherStringRepository: Decoder[Either[String, Repository]] = deriveConfiguredDecoderFullChecks implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/engines/docker/package.scala b/src/main/scala/io/viash/engines/docker/package.scala index 0a732628c..abaeed31d 100644 --- a/src/main/scala/io/viash/engines/docker/package.scala +++ b/src/main/scala/io/viash/engines/docker/package.scala @@ -18,7 +18,6 @@ package io.viash.engines import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object docker { import io.viash.helpers.circe._ diff --git a/src/main/scala/io/viash/engines/package.scala b/src/main/scala/io/viash/engines/package.scala index e6ef6f1c0..27e35a11f 100644 --- a/src/main/scala/io/viash/engines/package.scala +++ b/src/main/scala/io/viash/engines/package.scala @@ -23,7 +23,6 @@ import cats.syntax.functor._ package object engines { import io.viash.helpers.circe._ - import io.viash.engines.requirements.{decodeRequirements, encodeRequirements} implicit val encodeDockerEngine: Encoder.AsObject[DockerEngine] = deriveConfiguredEncoder diff --git a/src/main/scala/io/viash/engines/requirements/package.scala b/src/main/scala/io/viash/engines/requirements/package.scala index d3acf176e..c6ce02296 100644 --- a/src/main/scala/io/viash/engines/requirements/package.scala +++ b/src/main/scala/io/viash/engines/requirements/package.scala @@ -18,7 +18,6 @@ package io.viash.engines import io.circe.{Decoder, Encoder, Json} -// import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen package object requirements { diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index fb2475461..fffb80e45 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -39,8 +39,6 @@ import io.viash.config.Author import io.viash.config.ComputationalRequirements import io.viash.config.Links import io.viash.config.References -import scala.compiletime.{ codeOf, constValue, erasedValue, error, summonFrom, summonInline } -import scala.annotation.Annotation object CollectedSchemas { private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index f799969d5..69258fffa 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -17,7 +17,6 @@ package io.viash.schemas -// import scala.reflect.runtime.universe._ import io.viash.schemas.internalFunctionality import scala.annotation.Annotation From e7a05fbcb6344ad6c0b13bc11916462f37482c7e Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 28 Nov 2024 13:46:59 +0100 Subject: [PATCH 40/46] Revert circe to slightly older packages Both circe in general as circe-yaml need to be reverted to maximum these versions Circe made changes in 0.14.8 broke history but was supposed to be fixed in the meanwhile Circe-yaml also influences this through the set dependencies --- build.sbt | 4 ++-- .../circe/DeriveConfiguredDecoderWithValidationCheck.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 86546ad55..d163c8345 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,7 @@ libraryDependencies ++= Seq( "dev.optics" %% "monocle-macro" % "3.1.0" ) -val circeVersion = "0.14.10" +val circeVersion = "0.14.7" libraryDependencies ++= Seq( "io.circe" %% "circe-core", @@ -27,7 +27,7 @@ libraryDependencies ++= Seq( libraryDependencies ++= Seq( "io.circe" %% "circe-optics" % "0.15.0", - "io.circe" %% "circe-yaml" % "0.16.0", + "io.circe" %% "circe-yaml" % "0.15.2", ) scalacOptions ++= Seq("-unchecked", "-deprecation", "-explain") diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index f6ddeafa8..1eaf4ea6f 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -51,7 +51,7 @@ object DeriveConfiguredDecoderWithValidationCheck { // Validate the json can correctly converted to the required type by actually converting it. // Throw an exception when the conversion fails. - inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A], inline configuration: Configuration): Boolean = { + inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A], inline configuration: Configuration): Boolean = { val d = deriveConfiguredDecoder[A] // val v = d(pred) // TODO not entirely sure why this is needed instead of just doing `val v = d(pred)` From 25d5e41699bf101dc4b2e4b94001999ff7334e00 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Dec 2024 11:28:44 +0100 Subject: [PATCH 41/46] cleanup status enum circe code & switch Logger to use enums too --- src/main/scala/io/viash/config/package.scala | 18 ++----- .../scala/io/viash/helpers/BuildStatus.scala | 5 -- src/main/scala/io/viash/helpers/Logger.scala | 54 ++++++++----------- 3 files changed, 25 insertions(+), 52 deletions(-) diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index 5c3c4302d..0e4019b1e 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -29,6 +29,7 @@ import config.Status._ import config.arguments._ import io.circe.DecodingFailure import io.circe.DecodingFailure.Reason.CustomReason +import io.circe.derivation.{ConfiguredEnumEncoder, ConfiguredEnumDecoder} package object config { import io.viash.helpers.circe._ @@ -288,21 +289,10 @@ package object config { implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks // encoder and decoder for Status, make string lowercase before decoding - implicit val encodeStatus: Encoder[Status] = Encoder.instance{ - case Enabled => Json.fromString("enabled") - case Disabled => Json.fromString("disabled") - case Deprecated => Json.fromString("deprecated") + implicit val encodeStatus: Encoder[Status] = ConfiguredEnumEncoder.derive(_.toLowerCase()) + implicit val decodeStatus: Decoder[Status] = ConfiguredEnumDecoder.derive[Status](_.toLowerCase()).prepare { + _.withFocus(_.mapString(_.toLowerCase())) } - implicit val decodeStatus: Decoder[Status] = Decoder.instance(cursor => - for { - str <- cursor.as[String] - status <- str.toLowerCase() match - case "enabled" => Right(Enabled) - case "disabled" => Right(Disabled) - case "deprecated" => Right(Deprecated) - case other => Left(DecodingFailure(CustomReason(s"$other is not a valid Status"), cursor)) - } yield status - ) implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks diff --git a/src/main/scala/io/viash/helpers/BuildStatus.scala b/src/main/scala/io/viash/helpers/BuildStatus.scala index d4a418cd8..65df27788 100644 --- a/src/main/scala/io/viash/helpers/BuildStatus.scala +++ b/src/main/scala/io/viash/helpers/BuildStatus.scala @@ -17,11 +17,6 @@ package io.viash.helpers.status -// object BuildStatus extends Enumeration { -// type BuildStatus = Value -// val ParseError, Disabled, BuildError, TestError, TestMissing, Success = Value -// } - sealed trait Status { val isError: Boolean val color: String diff --git a/src/main/scala/io/viash/helpers/Logger.scala b/src/main/scala/io/viash/helpers/Logger.scala index 81d1c7f1a..8a2e5589c 100644 --- a/src/main/scala/io/viash/helpers/Logger.scala +++ b/src/main/scala/io/viash/helpers/Logger.scala @@ -19,38 +19,26 @@ package io.viash.helpers import scala.io.AnsiColor -object LoggerLevel extends Enumeration { - type Level = Value - val Error = Value(3) - val Warn = Value(4) - val Info = Value(6) - val Debug = Value(7) - val Trace = Value(8) +enum LoggerLevel(val level: Int): + case Error extends LoggerLevel(3) + case Warn extends LoggerLevel(4) + case Info extends LoggerLevel(6) + case Debug extends LoggerLevel(7) + case Trace extends LoggerLevel(8) - val Success = Value(5) // Should have been 6, same as Info. Luckely we had a spare spot between 4 and 6 + case Success extends LoggerLevel(5) // Should have been 6, same as Info. Luckely we had a spare spot between 4 and 6 - def fromString(level: String): Value = { - level match { - case "error" => Error - case "warn" => Warn - case "info" => Info - case "debug" => Debug - case "trace" => Trace - case _ => throw new IllegalArgumentException(level) - } - } +object LoggerLevel { + def fromString(level: String): LoggerLevel = LoggerLevel.valueOf(level.toLowerCase.capitalize) } -object LoggerOutput extends Enumeration { - type Output = Value - val StdOut = Value(1) - val StdErr = Value(2) -} +enum LoggerOutput: + case StdOut, StdErr /** Partial logging facade styled alike SLF4J. * Used Grizzled slf4j as further inspiration and basis. */ -class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boolean) { +class Logger(val name: String, val level: LoggerLevel, val useColor: Boolean) { import LoggerOutput._ import LoggerLevel._ @@ -74,9 +62,9 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole @inline final def traceOut(msg: => Any): Unit = _logOut(Trace, msg) @inline final def successOut(msg: => Any): Unit = _logOut(Success, msg) - @inline final def isEnabled(level: Level): Boolean = this.level >= level + @inline final def isEnabled(level: LoggerLevel): Boolean = this.level.level >= level.level - @inline private def _colorString(level: Level): String = + @inline private def _colorString(level: LoggerLevel): String = level match { case Error => AnsiColor.RED case Warn => AnsiColor.YELLOW @@ -87,7 +75,7 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole case Success => AnsiColor.GREEN } - @inline private def _log(level: Level, msg: => Any): Unit = { + @inline private def _log(level: LoggerLevel, msg: => Any): Unit = { if (!isEnabled(level)) return if (useColor) @@ -96,7 +84,7 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole Console.err.println(msg.toString()) } - @inline private def _logOut(level: Level, msg: => Any): Unit = { + @inline private def _logOut(level: LoggerLevel, msg: => Any): Unit = { if (!isEnabled(level)) return if (useColor) @@ -105,7 +93,7 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole Console.out.println(msg.toString()) } - @inline def log(out: Output, level: Level, color: String, msg: => Any): Unit = { + @inline def log(out: LoggerOutput, level: LoggerLevel, color: String, msg: => Any): Unit = { if (!isEnabled(level)) return val printer = @@ -148,7 +136,7 @@ trait Logging { protected def traceOut(msg: => Any): Unit = logger.traceOut(msg) protected def successOut(msg: => Any): Unit = logger.successOut(msg) - protected def log(out: LoggerOutput.Output, level: LoggerLevel.Level, color: String, msg: => Any): Unit = logger.log(out, level, color, msg) + protected def log(out: LoggerOutput, level: LoggerLevel, color: String, msg: => Any): Unit = logger.log(out, level, color, msg) } object Logger { @@ -156,15 +144,15 @@ object Logger { val rootLoggerName = "Viash-root-logger" - def apply(name: String, level: LoggerLevel.Level, useColor: Boolean): Logger = new Logger(name, level, useColor) + def apply(name: String, level: LoggerLevel, useColor: Boolean): Logger = new Logger(name, level, useColor) def apply(name: String): Logger = new Logger(name, getLoggerLevel(name), useColor) def apply(cls: Class[_]): Logger = apply(cls.getName) def apply[C: ClassTag](): Logger = apply(classTag[C].runtimeClass.getName) def rootLogger = apply(rootLoggerName) - object UseLevelOverride extends util.DynamicVariable[LoggerLevel.Level](LoggerLevel.Info) - def getLoggerLevel(name: String): LoggerLevel.Level = { + object UseLevelOverride extends util.DynamicVariable[LoggerLevel](LoggerLevel.Info) + def getLoggerLevel(name: String): LoggerLevel = { if (name != rootLoggerName) // prevent constructor loop rootLogger.debug(s"GetLoggerLevel for $name") From 9a4b0ba97f43addb613ed2d3bd5b075aca31e4f6 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Dec 2024 11:44:42 +0100 Subject: [PATCH 42/46] Minor code cleanup --- src/main/scala/io/viash/config/Status.scala | 5 ----- .../circe/DeriveConfiguredDecoderWithDeprecationCheck.scala | 1 + .../circe/DeriveConfiguredDecoderWithValidationCheck.scala | 5 +++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/scala/io/viash/config/Status.scala b/src/main/scala/io/viash/config/Status.scala index bb2d1d9b3..65d8bb69c 100644 --- a/src/main/scala/io/viash/config/Status.scala +++ b/src/main/scala/io/viash/config/Status.scala @@ -19,8 +19,3 @@ package io.viash.config enum Status: case Enabled, Disabled, Deprecated - // type Status = Value - // val Enabled = Value("enabled") - // val Disabled = Value("disabled") - // val Deprecated = Value("deprecated") - diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 2e0857207..b26c6195c 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -26,6 +26,7 @@ import io.viash.helpers.* object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { + // This method doesn't use any mirroring, so it can be called from multiple inlined validators without needing inlining. private def memberDeprecationCheck( name: String, history: List[CursorOp], diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala index 1eaf4ea6f..a4cf280ca 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -27,7 +27,8 @@ import io.viash.helpers.{typeOf, fieldsOf} object DeriveConfiguredDecoderWithValidationCheck { - private def validator_static_error(pred: HCursor, validFields: List[String], typeOf: String)(error: DecodingFailure): Boolean = { + // This method doesn't use any mirroring, so it can be called from multiple inlined validators without needing inlining. + private def validatorError(pred: HCursor, validFields: List[String], typeOf: String)(error: DecodingFailure): Boolean = { val usedFields = pred.value.asObject.map(_.keys.toSeq) val invalidFields = usedFields.map(_.diff(validFields)) @@ -62,7 +63,7 @@ object DeriveConfiguredDecoderWithValidationCheck { } v.fold( - validator_static_error(pred, fieldsOf[A], typeOf[A]), + validatorError(pred, fieldsOf[A], typeOf[A]), _ => true) } From d1b0920611a9cae77787868c68c13e0b4fa734b5 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Dec 2024 12:02:09 +0100 Subject: [PATCH 43/46] Update src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala --- .../circe/DeriveConfiguredDecoderWithDeprecationCheck.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index b26c6195c..1dbf991e3 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -100,7 +100,7 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { cursor // return unchanged json info } - // // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here +// Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] .prepare( checkDeprecation[A] ) } From bf1c76baf793cbc54ae27d55c3026a98753986d9 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Thu, 5 Dec 2024 12:03:13 +0100 Subject: [PATCH 44/46] Update src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala --- .../circe/DeriveConfiguredDecoderWithDeprecationCheck.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 1dbf991e3..b1b4434a9 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -100,7 +100,7 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { cursor // return unchanged json info } -// Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here + // Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A] .prepare( checkDeprecation[A] ) } From e514e9cefd3e8d8ae952b50f4e0f8f8d5fea0879 Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Mon, 9 Dec 2024 11:04:15 +0100 Subject: [PATCH 45/46] Add changelog entry, fixes for website output --- CHANGELOG.md | 5 +++++ src/main/scala/io/viash/helpers/SysEnv.scala | 13 +++++++++---- .../scala/io/viash/schemas/CollectedSchemas.scala | 12 +----------- .../scala/io/viash/schemas/ParameterSchema.scala | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a10ded3b..6a54a597c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ TODO add summary * `RRequirements`: Allow single quotes to be used again in the `.script` field (PR #771). +* `scala`: Update Scala to Scala 3 (PR #759). + For most of the code, this was a minor update, so no breaking changes are expected. + The biggest change is how the exporting of the schema is done, but this has no impact on the user. + However, switching to Scala 3 allows for additional features and improvements in the future. + ## BUG FIXES * `config build`: Fix a bug where a missing main script would cause a stack trace instead of a proper error message (PR #776). diff --git a/src/main/scala/io/viash/helpers/SysEnv.scala b/src/main/scala/io/viash/helpers/SysEnv.scala index 580f0bf61..a016012a5 100644 --- a/src/main/scala/io/viash/helpers/SysEnv.scala +++ b/src/main/scala/io/viash/helpers/SysEnv.scala @@ -19,17 +19,22 @@ package io.viash.helpers import io.viash.schemas._ +trait SysEnvTrait { + def viashHome: String + def viashVersion: Option[String] +} + @nameOverride("EnvironmentVariables") @description("Viash checks several environment variables during operation.") @documentFully -trait SysEnvTrait { +case class SysEnvCC( @nameOverride("VIASH_HOME") @description( """If `VIASH_HOME` is not defined, the fallback `HOME`/.viash is used. | |Location where specific downloaded versions of Viash will be cached and run from. |""") - def viashHome: String + viashHome: String, @nameOverride("VIASH_VERSION") @description( @@ -40,8 +45,8 @@ trait SysEnvTrait { """VIASH_VERSION=0.7.0 viash ns build""", "sh" ) - def viashVersion: Option[String] -} + viashVersion: Option[String] +) extends SysEnvTrait object SysEnv extends SysEnvTrait { diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index fffb80e45..b50d99272 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -67,7 +67,7 @@ object CollectedSchemas { getMembers[Config](), getMembers[PackageConfig](), getMembers[BuildInfo](), - getMembers[SysEnvTrait](), + getMembers[SysEnvCC](), getMembers[Functionality](), getMembers[Author](), @@ -144,16 +144,6 @@ object CollectedSchemas { ) } - private def trimTypeName(s: String) = { - // first: io.viash.helpers.data_structures.OneOrMore[String] -> OneOrMore[String] - // second: List[io.viash.platforms.requirements.Requirements] -> List[Requirements] - // third: Either[String,io.viash.functionality.dependencies.Repository] -> Either[String,Repository] - s - .replaceAll("""^(\w*\.)*""", "") - .replaceAll("""(\w*)\[[\w\.]*?(\w*)(\[_\])?\]""", "$1[$2]") - .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]") - } - val fullData = memberData_part1.part ++ memberData_part2.part ++ memberData_part3.part // Main call for documentation output diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index 69258fffa..de040d2c9 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -100,8 +100,8 @@ object ParameterSchema { } val description = annotations.collectFirst({case (name, value) if name.endsWith("description") => value.head}) - val example = annotations.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) - val exampleWithDescription = annotations.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) + val example = annotations.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)).reverse + val exampleWithDescription = annotations.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)).reverse val examples = example ::: exampleWithDescription match { case l if l.length > 0 => Some(l) case _ => None From 4c8cb7de78f65eba0c3bba54e0819abaea2cfc0f Mon Sep 17 00:00:00 2001 From: Hendrik Cannoodt Date: Mon, 9 Dec 2024 11:30:28 +0100 Subject: [PATCH 46/46] Remove testbench for method that was removed --- .../scala/io/viash/schema/SchemaTest.scala | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/test/scala/io/viash/schema/SchemaTest.scala b/src/test/scala/io/viash/schema/SchemaTest.scala index f059e53b6..2f13f77b8 100644 --- a/src/test/scala/io/viash/schema/SchemaTest.scala +++ b/src/test/scala/io/viash/schema/SchemaTest.scala @@ -9,27 +9,7 @@ import io.viash.helpers.Logger class SchemaTest extends AnyFunSuite with BeforeAndAfterAll with PrivateMethodTester{ Logger.UseColorOverride.value = Some(false) - - test("Check type name trimming") { - val checks = Map ( - "foo" -> "foo", - "foo.bar" -> "bar", - "foo.bar.baz" -> "baz", - "foo.bar[baz]" -> "bar[baz]", - "foo[bar.baz]" -> "foo[baz]", - "foo[bar,baz]" -> "foo[bar,baz]", - "foo[bar.baz,quux]" -> "foo[baz,quux]", - "foo[bar,baz.quux]" -> "foo[bar,quux]" - ) - - val trimTypeName = PrivateMethod[String](Symbol("trimTypeName")) - - for ((k, v) <- checks) { - val res = CollectedSchemas invokePrivate trimTypeName(k) - assert(res == v, s"$k -> $v != $res") - } - } - + test("All schema class val members should be annotated") { val nonAnnotated = CollectedSchemas.getAllNonAnnotated