Merge pull request #782 from viash-io/feature/scope_publishmode

Add Scope & ScopeEnum classes. Add .scope in config

CHANGELOG.md

@@ -6,6 +6,8 @@ TODO add summary
 
 * `Nextflow` runner: allow emitting multiple output channels (PR #736).
 
+* `Scope`: Add a `scope` field to the config (PR #782). This allows tuning how the components is built for release.
+
 ## MINOR CHANGES
 
 * `viash-hub`: Change the url for viash-hub Git access to packages.viash-hub.com (PR #774).

src/main/scala/io/viash/ViashNamespace.scala

@@ -33,6 +33,7 @@ import io.viash.helpers.LoggerOutput
 import io.viash.helpers.LoggerLevel
 import io.viash.runners.Runner
 import io.viash.config.AppliedConfig
+import io.viash.config.{ScopeEnum, Scope}
 
 object ViashNamespace extends Logging {
 
@@ -52,20 +53,31 @@ object ViashNamespace extends Logging {
         list.foreach(f)
   }
 
-  def targetOutputPath(targetDir: String, runnerId: String, config: Config): String =
-    targetOutputPath(targetDir, runnerId, config.namespace, config.name)
+  def targetOutputPath(targetDir: String, runnerId: String, config: Config): String = {
+    val scope = config.scope match {
+      case Left(value) => Scope(value)
+      case Right(value) => value
+    }
+    targetOutputPath(targetDir, runnerId, scope.target, config.namespace, config.name)
+  }
 
   def targetOutputPath(
     targetDir: String,
     runnerId: String,
+    scope: ScopeEnum,
     namespace: Option[String],
     name: String
   ): String = {
     val nsStr = namespace match {
       case Some(ns) => ns + "/"
       case None => ""
     }
-    s"$targetDir/$runnerId/$nsStr$name"
+    val scopeStr = scope match {
+      case ScopeEnum.Test => "_test/"
+      case ScopeEnum.Private => "_private/"
+      case ScopeEnum.Public => ""
+    }
+    s"$targetDir/$scopeStr$runnerId/$nsStr$name"
   }
 
   def build(

src/main/scala/io/viash/config/Config.scala

@@ -242,6 +242,15 @@ case class Config(
   @since("Viash 0.6.0")
   @default("Enabled")
   status: Status = Status.Enabled,
+
+  @description(
+    """Defines the scope of the component.
+      |`test`: only available during testing; components aren't published.
+      |`private`: only meant for internal use within a workflow or other component.
+      |`public`: core component or workflow meant for general use.""")
+  @since("Viash 0.9.1")
+  @default("public")
+  scope: Either[ScopeEnum, Scope] = Left(ScopeEnum.Public),
 
   @description(
     """@[Computational requirements](computational_requirements) related to running the component. 
@@ -736,16 +745,21 @@ object Config extends Logging {
     val conf4 = conf3.copy(
       package_config = viashPackage
     )
-
-    if (!addOptMainScript) {
-      return conf4
-    }
-
+
     /* CONFIG 5: add main script if config is stored inside script */
     // add info and additional resources
-    val conf5 = resourcesLens.modify(optScript.toList ::: _)(conf4)
+    val conf5 = addOptMainScript match {
+      case true => resourcesLens.modify(optScript.toList ::: _)(conf4)
+      case false => conf4
+    }
+
+    /* CONFIG 6: Finalize Scope */
+    val conf6 = scopeLens.modify {
+      case Left(scope) => Right(Scope(scope))
+      case right => right
+    }(conf5)
 
-    conf5
+    conf6
   }
 
   def readConfigs(

src/main/scala/io/viash/config/Scope.scala

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2020  Data Intuitive
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package io.viash.config
+
+import io.viash.schemas.description
+
+enum ScopeEnum {
+  case Test, Private, Public
+}
+
+@description(
+  """Defines the scope of the component.
+    |`test`: only available during testing; components aren't published.
+    |`private`: only meant for internal use within a workflow or other component.
+    |`public`: core component or workflow meant for general use.""")
+case class Scope(
+  @description(
+    """test: image is only used during testing and is transient
+      |private: image is published in the registry
+      |public: image is published in the registry""")
+  image: ScopeEnum,
+  @description(
+    """test: target folder is only used during testing and is transient
+      |private: target folder can be published in target/private or target/dependencies/private
+      |public: target is published in target/executable or target/nextflow"""
+  )
+  target: ScopeEnum,
+)
+
+object Scope {
+  def apply(scopeValue: ScopeEnum): Scope = {
+    Scope(scopeValue, scopeValue)
+  }
+}

src/main/scala/io/viash/config/dependencies/Dependency.scala

@@ -23,6 +23,7 @@ import io.viash.schemas._
 import java.nio.file.Files
 import io.viash.ViashNamespace
 import io.viash.exceptions.MissingBuildYamlException
+import io.viash.config.ScopeEnum
 
 @description(
   """Specifies a Viash component (script or executable) that should be made available for the code defined in the component.
@@ -92,6 +93,9 @@ case class Dependency(
   @internalFunctionality
   @description("Location of the dependency component artifacts are written ready to be used.")
   writtenPath: Option[String] = None,
+
+  @internalFunctionality
+  internalDependencyTargetScope: ScopeEnum = ScopeEnum.Public
 ) {
   if (alias.isDefined) {
     // check functionality name
@@ -119,7 +123,7 @@ case class Dependency(
     if (isLocalDependency) {
       // Local dependency so it will only exist once the component is built.
       // TODO improve this, for one, the runner id should be dynamic
-      Some(ViashNamespace.targetOutputPath("", "executable", None, name))
+      Some(ViashNamespace.targetOutputPath("", "executable", internalDependencyTargetScope, None, name))
     } else {
       // Previous existing dependency. Use the location of the '.build.yaml' to determine the relative location.
       val relativePath = Dependency.getRelativePath(fullPath, Paths.get(workRepository.get.localPath))

src/main/scala/io/viash/config/package.scala

@@ -294,6 +294,16 @@ package object config {
     _.withFocus(_.mapString(_.toLowerCase()))
   }
 
+  // encoder and decoder for ScopeEnum, make string lowercase before decoding
+  implicit val encodeScopeEnum: Encoder[ScopeEnum] = ConfiguredEnumEncoder.derive(_.toLowerCase())
+  implicit val decodeScopeEnum: Decoder[ScopeEnum] = ConfiguredEnumDecoder.derive[ScopeEnum](_.toLowerCase()).prepare {
+    _.withFocus(_.mapString(_.toLowerCase()))
+  }
+
+  // encoder and decoder for Scope
+  implicit val encodeScope: Encoder.AsObject[Scope] = deriveConfiguredEncoder
+  implicit val decodeScope: Decoder[Scope] = deriveConfiguredDecoderFullChecks
+
   implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict
   implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks
 

src/main/scala/io/viash/helpers/DependencyResolver.scala

@@ -32,6 +32,7 @@ import io.viash.ViashNamespace
 import io.viash.config.resources.NextflowScript
 import io.viash.exceptions.MissingDependencyException
 import io.viash.helpers.circe.Convert
+import io.viash.config.ScopeEnum
 
 object DependencyResolver extends Logging {
 
@@ -94,14 +95,19 @@ object DependencyResolver extends Logging {
 
         val config =
           if (dep.isLocalDependency) {
-            findLocalConfig(repo.localPath.toString(), namespaceConfigs, dep.name, runnerId)
+            val t = findLocalConfig(repo.localPath.toString(), namespaceConfigs, dep.name, runnerId)
+            t.map(t => (t._1, t._2, Some(t._3)))
           } else {
-            findRemoteConfig(repo.localPath.toString(), dep.name, runnerId)
+            val t = findRemoteConfig(repo.localPath.toString(), dep.name, runnerId)
+            t.map(t => (t._1, t._2, Option.empty[Config]))
           }
 
+        val internalDependencyTargetScope = config.flatMap(_._3).flatMap(_.scope.toOption).map(_.target).getOrElse(ScopeEnum.Public)
+
         dep.copy(
           foundConfigPath = config.map(_._1),
-          configInfo = config.map(_._2).getOrElse(Map.empty)
+          configInfo = config.map(_._2).getOrElse(Map.empty),
+          internalDependencyTargetScope = internalDependencyTargetScope
         )
       }
       )(config3)
@@ -121,7 +127,7 @@ object DependencyResolver extends Logging {
       if (dep.isLocalDependency) {
         // Dependency solving will be done by building the component and dependencies of that component will be handled there.
         // However, we have to fill in writtenPath. This will be needed when this built component is used as a dependency and we have to resolve dependencies of dependencies.
-        val writtenPath = ViashNamespace.targetOutputPath(output, runnerId, None, dep.name)
+        val writtenPath = ViashNamespace.targetOutputPath(output, runnerId, dep.internalDependencyTargetScope, None, dep.name)
         dep.copy(writtenPath = Some(writtenPath))
       } else {
         // copy the dependency to the output folder
@@ -144,7 +150,7 @@ object DependencyResolver extends Logging {
   }
 
   // Find configs from the local repository. These still need to be built so we have to deduce the information we want.
-  def findLocalConfig(targetDir: String, namespaceConfigs: List[Config], name: String, runnerId: Option[String]): Option[(String, Map[String, String])] = {
+  def findLocalConfig(targetDir: String, namespaceConfigs: List[Config], name: String, runnerId: Option[String]): Option[(String, Map[String, String], Config)] = {
 
     val config = namespaceConfigs.filter{ c => 
         val fullName = c.namespace.fold("")(n => n + "/") + c.name
@@ -177,7 +183,7 @@ object DependencyResolver extends Logging {
         ("name" -> c.name),
         ("namespace" -> c.namespace.getOrElse(""))
       )
-      (path, map ++ map2)
+      (path, map ++ map2, c)
     }
   }
 

src/main/scala/io/viash/lenses/ConfigLenses.scala

@@ -38,6 +38,7 @@ object ConfigLenses {
   val keywordsLens = GenLens[Config](_.keywords)
   val licenseLens = GenLens[Config](_.license)
   val linksLens = GenLens[Config](_.links)
+  val scopeLens = GenLens[Config](_.scope)
 
   val linksRepositoryLens = linksLens andThen repositoryLens
   val linksDockerRegistryLens = linksLens andThen LinksLenses.dockerRegistryLens

src/main/scala/io/viash/schemas/CollectedSchemas.scala

@@ -39,6 +39,7 @@ import io.viash.config.Author
 import io.viash.config.ComputationalRequirements
 import io.viash.config.Links
 import io.viash.config.References
+import io.viash.config.Scope
 
 object CollectedSchemas {
   private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true)
@@ -75,6 +76,7 @@ object CollectedSchemas {
       getMembers[ArgumentGroup](),
       getMembers[Links](),
       getMembers[References](),
+      getMembers[Scope](),
 
       getMembers[Runner](),
       getMembers[ExecutableRunner](),

src/main/scala/io/viash/schemas/JsonSchema.scala

@@ -287,15 +287,17 @@ object JsonSchema {
         "DockerSetupStrategy" -> createEnum(DockerSetupStrategy.objs.map(obj => obj.id).toSeq, Some("The Docker setup strategy to use when building a container."), Some("TODO add descriptions to different strategies")),
         "Direction" -> createEnum(Seq("input", "output"), Some("Makes this argument an `input` or an `output`, as in does the file/folder needs to be read or written. `input` by default."), None),
         "Status" -> createEnum(Seq("enabled", "disabled", "deprecated"), Some("Allows setting a component to active, deprecated or disabled."), None),
-        "DoubleStrings" -> createEnum(Seq("+infinity", "-infinity", "nan"), None, None)
+        "DoubleStrings" -> createEnum(Seq("+infinity", "-infinity", "nan"), None, None),
+        "ScopeEnum" -> createEnum(Seq("test", "private", "public"), Some("The scope of the component. `public` by default."), None),
       )
     } else {
       Seq(
         "DockerSetupStrategy" -> createEnum(DockerSetupStrategy.map.keys.toSeq, Some("The Docker setup strategy to use when building a container."), Some("TODO add descriptions to different strategies")),
         "Direction" -> createEnum(Seq("input", "output"), Some("Makes this argument an `input` or an `output`, as in does the file/folder needs to be read or written. `input` by default."), None),
         "Status" -> createEnum(Seq("enabled", "disabled", "deprecated"), Some("Allows setting a component to active, deprecated or disabled."), None),
         "DockerResolveVolume" -> createEnum(Seq("manual", "automatic", "auto", "Manual", "Automatic", "Auto"), Some("Enables or disables automatic volume mapping. Enabled when set to `Automatic` or disabled when set to `Manual`. Default: `Automatic`"), Some("TODO make fully case insensitive")),
-        "DoubleStrings" -> createEnum(Seq("+.inf", "+inf", "+infinity", "positiveinfinity", "positiveinf", "-.inf", "-inf", "-infinity", "negativeinfinity", "negativeinf", ".nan", "nan"), None, None)
+        "DoubleStrings" -> createEnum(Seq("+.inf", "+inf", "+infinity", "positiveinfinity", "positiveinf", "-.inf", "-inf", "-infinity", "negativeinfinity", "negativeinf", ".nan", "nan"), None, None),
+        "ScopeEnum" -> createEnum(Seq("test", "private", "public"), Some("The scope of the component. `public` by default."), None),
       )
     }
 

src/test/scala/io/viash/config/ConfigTest.scala

@@ -14,6 +14,7 @@ import io.viash.engines.NativeEngine
 import io.viash.runners.ExecutableRunner
 import io.viash.helpers.IO
 import io.viash.helpers.status
+import io.viash.ConfigDeriver
 
 class ConfigTest extends AnyFunSuite with BeforeAndAfterAll {
   Logger.UseColorOverride.value = Some(false)
@@ -22,6 +23,10 @@ class ConfigTest extends AnyFunSuite with BeforeAndAfterAll {
   private val tempFolStr = temporaryFolder.toString
 
   private val nsPath = getClass.getResource("/testns/").getPath
+
+  private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath
+  private val temporaryConfigFolder = IO.makeTemp(s"viash_${this.getClass.getName}_")
+  private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryConfigFolder)
 
   val infoJson = Yaml("""
     |foo:
@@ -131,7 +136,55 @@ class ConfigTest extends AnyFunSuite with BeforeAndAfterAll {
     assert(configs.filter(_.status == Some(status.ParseError)).length == 2, "Expect 2 failed component")
   }
 
-  // TODO: expand functionality tests
+  test("Test default scope value") {
+    val newConfigFilePath = configDeriver.derive(Nil, "default_scope")
+    val newConfig = Config.read(newConfigFilePath)
+
+    assert(newConfig.scope.isRight)
+    val scope = newConfig.scope.toOption.get
+    assert(scope.image == ScopeEnum.Public)
+    assert(scope.target == ScopeEnum.Public)
+  }
+
+  test("Test public scope value") {
+    val newConfigFilePath = configDeriver.derive(""".scope := "public"""", "public_scope")
+    val newConfig = Config.read(newConfigFilePath)
+
+    assert(newConfig.scope.isRight)
+    val scope = newConfig.scope.toOption.get
+    assert(scope.image == ScopeEnum.Public)
+    assert(scope.target == ScopeEnum.Public)
+  }
+
+  test("Test private scope value") {
+    val newConfigFilePath = configDeriver.derive(""".scope := "private"""", "private_scope")
+    val newConfig = Config.read(newConfigFilePath)
+
+    assert(newConfig.scope.isRight)
+    val scope = newConfig.scope.toOption.get
+    assert(scope.image == ScopeEnum.Private)
+    assert(scope.target == ScopeEnum.Private)
+  }
+
+  test("Test test scope value") {
+    val newConfigFilePath = configDeriver.derive(""".scope := "test"""", "test_scope")
+    val newConfig = Config.read(newConfigFilePath)
+
+    assert(newConfig.scope.isRight)
+    val scope = newConfig.scope.toOption.get
+    assert(scope.image == ScopeEnum.Test)
+    assert(scope.target == ScopeEnum.Test)
+  }
+
+  test("Test scope value with different image and target") {
+    val newConfigFilePath = configDeriver.derive(""".scope := {image: "test", target: "private"}""", "custom_scope")
+    val newConfig = Config.read(newConfigFilePath)
+
+    assert(newConfig.scope.isRight)
+    val scope = newConfig.scope.toOption.get
+    assert(scope.image == ScopeEnum.Test)
+    assert(scope.target == ScopeEnum.Private)
+  }
 
   override def afterAll(): Unit = {
     IO.deleteRecursively(temporaryFolder)