diff --git a/.github/workflows/ns_test.yml b/.github/workflows/ns_test.yml
index 336f786cf..f76afcdc7 100644
--- a/.github/workflows/ns_test.yml
+++ b/.github/workflows/ns_test.yml
@@ -9,12 +9,15 @@ jobs:
steps:
- uses: actions/checkout@v4
- - name: Set up sbt
+ - name: Set up java
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '11'
+ - name: Set up sbt
+ uses: sbt/setup-sbt@v1
+
- name: Build viash
run: |
echo "${HOME}/.local/bin" >> $GITHUB_PATH
diff --git a/.github/workflows/sbt_test.yml b/.github/workflows/sbt_test.yml
index 1ebcb6359..d0c6d18d8 100644
--- a/.github/workflows/sbt_test.yml
+++ b/.github/workflows/sbt_test.yml
@@ -24,12 +24,6 @@ jobs:
- uses: viash-io/viash-actions/update-docker-engine@v5
if: runner.os == 'Linux'
- - name: Set up Nextflow
- if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }}
- uses: nf-core/setup-nextflow@v1
- with:
- version: ${{ matrix.java.nxf_ver }}
-
- name: Set up R
uses: r-lib/actions/setup-r@v2
with:
@@ -42,18 +36,14 @@ jobs:
processx
testthat
- - name: Set up java & sbt
+ - name: Set up java
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: ${{ matrix.java.ver }}
- - name: Set up sbt specifically on macOS on arm64 if needed
- if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64'}}
- run: |
- if ! command -v sbt &> /dev/null; then
- brew install sbt
- fi
+ - name: Set up sbt
+ uses: sbt/setup-sbt@v1
- name: Set up Scala
run: |
@@ -69,6 +59,12 @@ jobs:
with:
python-version: '3.x'
+ - name: Set up Nextflow
+ if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }}
+ uses: nf-core/setup-nextflow@v1
+ with:
+ version: ${{ matrix.java.nxf_ver }}
+
- name: Run tests
run: |
if [[ "${{ matrix.config.name }}" =~ ^ubuntu.*$ ]] && [[ "${{ matrix.java.run_coverage }}" == "true" ]]; then
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 580e07a9f..efa479144 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,118 +2,135 @@
TODO add summary
-# Viash 0.9.0-RC7 (2024-08-13): General bugfixes and improvements
+## NEW FEATURES
-These are bug fixes and other improvements that solve some edge case issues and improve the overall user experience and usability of Viash.
+* `Nextflow` runner: allow emitting multiple output channels (PR #736).
-## BREAKING CHANGES
+* `Scope`: Add a `scope` field to the config (PR #782). This allows tuning how the components is built for release.
-* `NextflowPlatform`: Swap the order of execution of `runIf` and `filter` when calling `.run()`. This means that `runIf` is now executed before `filter` (PR #660).
+## MINOR CHANGES
-## NEW FUNCTIONALITY
+* `viash-hub`: Change the url for viash-hub Git access to packages.viash-hub.com (PR #774).
-* `ExecutableRunner`: Add a `---docker_image_id` flag to view the Docker image ID of a built executable (PR #741).
+* `RRequirements`: Allow single quotes to be used again in the `.script` field (PR #771).
-* `viash ns query`: Add a query filter that allows selecting a single component by its path in a namespace environment (PR #744).
+* `scala`: Update Scala to Scala 3 (PR #759).
+ For most of the code, this was a minor update, so no breaking changes are expected.
+ The biggest change is how the exporting of the schema is done, but this has no impact on the user.
+ However, switching to Scala 3 allows for additional features and improvements in the future.
-* `config schema`: Add `label` & `summary` fields for Config, PackageConfig, argument groups, and all argument types (PR #743).
+* `--help`: Component `--help` messages will now display what built in `---` options are available (PR #784).
-* `NextflowPlatform`: Added `runIf` functionality to `runEach` (PR #660).
+## BUG FIXES
-## MINOR CHANGES
+* `config build`: Fix a bug where a missing main script would cause a stack trace instead of a proper error message (PR #776).
+ The error message showed the path of the missing resource but it was easy to miss given the stack trace, besides it shouldn't have been a stack trace anyway.
+
+* `RRequirements`: Treat warnings as errors when installing R dependencies in Docker engines (PR #771).
-* `ExecutableRunner`: Add parameter `docker_automount_prefix` to allow for a custom prefix for automounted folders (PR #739).
+* `Nextflow` runner: fix false-positive error when output argument arguments `required: true`
+ are incorrectly flagged as missing input arguments (PR #778).
-* `ExecutableRunner`: Make Docker runtime arguments configurable via the `---docker_run_args` argument (PR #740).
+# Viash 0.9.0 (2024-09-03): Restructure platforms into runners and engines
-* `export json_schema`: Add `arguments` field to the `Config` schema (PR #755). Only for the non-strict version, the strict version of the viash config has these values merged into `argument_groups`.
+This release restructures the introduces changes to the Viash config:
+- The `platforms` field is split into `runners` and `engines`
+- The `.functionality` layer has been removed from the config and all fields have been moved to the top layer
-## BUG FIXES
+Changes are made to sanitize the built config output and include additional relevant meta data.
+The default `multiple_sep` has been changed from `:` to `;` to avoid conflicts with paths like `s3://foo/bar`.
-* `platforms`: Re-introduce the `--platform` and `--apply_platform` arguments to improve backwards compatibility (PR #725).
- When the argument is used, a deprecation warning message is printed on stderr.
- Cannot be used together with `--engine` or `--runner` and/or `--apply_engine` or `--apply_runner`.
+Implemented a proper way of caching dependency repositories. The cache is stored under `~/.viash/repositories`.
-* `nextflow_runner`: Fix refactoring error in the `findStates()` helper function (PR #733).
+## BREAKING CHANGES
-* `viash ns exec`: Fix "relative fields" outputting absolute paths (PR# 737). Additionally, improve path resolution when using the `--src` argument.
+* `runners` and `engines`: The usage of `platforms` is deprecated and instead these are split into `runners` and `engines` (PR #510).
+ The `platforms` field is still supported but will be removed in a future release.
+ In brief, the `native platform` became a `native engine` and `docker platform` became a `docker engine`.
+ Additionally, the `native platform` and `docker platform` became a `executable runner`, `nextflow platform` became a `nextflow runner`.
+ The fields of `docker platform` is split between `docker engine` and `docker runner`: `port`, `workdir`, `setup_strategy`, and `run_args` (set to `docker_run_args`) are captured by the `runner` as they define how the component is run. The other fields are captured by the `engine` as they define the environment in which the component is run. One exception is `chown` which is rarely set to false and is now always enabled.
-* `viash ns`: Fix viash tripping over its toes when it encounters multiple failed configs (PR #761). A dummy config was used as a placeholder, but it always used the name `failed`, so duplicate config names were generated, which we check for nowadays.
+* `arguments`: Merge arguments into argument_groups during a json decode prepare step (PR #574). The `--parse_argument_groups` option from `ns list` and `config view` is deprecated as it is now always enabled.
-* `bashwrapper`: Fix an issue where running `viash test` which builds the test docker container would ignore test failures but subsequential runs would work correctly (PR #754).
+* `arguments`: Change default `multiple_sep` from `:` to `;` to avoid conflicts with paths like `s3://foo/bar` (PR #645).
+ The previous behaviour of using `multiple_sep: ":"` can be achieved by adding a config mod to the `_viash.yaml`:
+ ```yaml
+ config_mods: |
+ .functionality.argument_groups[true].arguments[.multiple == true].multiple_sep := ":"
+ ```
-# Viash 0.9.0-RC6 (2024-06-17): Hotfix for docker image name generation
+* `functionality`: Remove the `functionality` layer from the config and move all fields to the top layer (PR #649).
-Fix an issue where docker image names were not generated correctly.
+* `computational requirements`: Use 1000-base units instead of 1024-base units for memory (PR #686). Additionally, the memory units `kib`, `mib`, `gib`, `tib`, and `pib` are added to support 1024-base definitions.
-## BUG FIXES
+* `NextflowEngine`: Swap the order of execution of `runIf` and `filter` when calling `.run()`. This means that `runIf` is now executed before `filter` (PR #660).
-* `docker_engine`: Fix a bug in how the namespace separator is handled (PR #722).
+## NEW FUNCTIONALITY
-# Viash 0.9.0-RC5 (2024-06-13): Improvements for CI
+* `export json_schema`: Add a `--strict` option to output a subset of the schema representing the internal structure of the Viash config (PR #564).
-Dependencies now use `vsh` as the default organization level. This means that the organization level is now optional in the `repo` field of the dependencies.
-Improved how the docker image name is generated to be more predictable.
+* `config view` and `ns list`: Do not output internal functionality fields (#564). Additionally, add a validation that no internal fields are present when reading a Viash config file.
-## MINOR CHANGES
+* `project config`: Add fields in the project config to specify default values for component config fields (PR #612). This allows for a more DRY approach to defining the same values for multiple components.
-* `resources_test`: This field is removed again from the `_viash.yaml` as it was decided to impliment this temporary functionality using the `info` field (PR #711).
+* `dependencies`: GitHub and ViashHub repositories now get properly cached (PR #699).
+ The cache is stored in the `~/.viash/repositories` directory using sparse-checkout to only fetch the necessary files.
+ During a build, the cache is checked for the repository and if it is found and still up-to-date, the repository is not cloned again and instead the cache is copied to a temporary folder where the files are checked out from the sparse-checkout.
-* `docker_engine`: Deprecate `registry`, `organization` and `tag` fields in the `docker_engine` (PR #712). Currently these are hardly ever used and instead the `image` field is used to specify the full image name.
+* `ExecutableRunner`: Add a `---docker_image_id` flag to view the Docker image ID of a built executable (PR #741).
-* `docker_engine`: Add `target_package` field to the `docker_engine` (PR #712). This field, together with the `target_organization` is used to specify the full built container image name. The fields use proper fallback for the values set in the component config and package config.
+* `viash ns query`: Add a query filter that allows selecting a single component by its path in a namespace environment (PR #744).
-* `organization`: Remove the `organization` field from the component config (PR #712). The value is now directly used by the `docker_engine` as a fallback from the `target_organization` field.
+* `config schema`: Add `label` & `summary` fields for Config, PackageConfig, argument groups, and all argument types (PR #743).
-## BUG FIXES
+* `NextflowEngine`: Added `runIf` functionality to `runEach` (PR #660).
-* `build_info`: Correctly set the `.build_info.executable` to `main.nf` when building a component with a Nextflow runner (PR #720).
+## MINOR CHANGES
-* `vsh organization`: ViashHub repositories now use `vsh` as the default organization (PR #718).
- Instead of having to specify `repo: vsh/repo_name`, you can now just specify `repo: repo_name`, which is now also the prefered way.
+* `testbenches`: Add testbenches for local dependencies (PR #565).
-* `testbenches`: Add a testbench to verify dependencies in dependencies from scratch (PR #721).
- The components are built from scratch and the dependencies are resolved from the local repositories.
+* `testbenches`: Refactor testbenches helper functions to uniformize them (PR #565).
-# Viash 0.9.0-RC4 (2024-05-29): Improvements for CI
+* `logging`: Preserve log order of StdOut and StdErr messages during reading configs in namespaces (PR #571).
-These are mainly improvements for issues highlighted by running Viash in a CI environment.
-Additionally, implemented a proper way of caching dependency repositories. The cache is stored under `~/.viash/repositories`.
+* `Java 21 support`: Update Scala to 2.13.12 and update dependencies (PR #602).
-## NEW FUNCTIONALITY
+* `project config`: Output the project config under the default name `ProjectConfig` instead of `Project` during schema export (PR #631). This is now important as the project config is now part of the component config. Previously this was overridden as the class name was `ViashProject` which was less descriptive.
-* `dependencies`: GitHub and ViashHub repositories now get properly cached (PR #699).
- The cache is stored in the `~/.viash/repositories` directory using sparse-checkout to only fetch the necessary files.
- During a build, the cache is checked for the repository and if it is found and still up-to-date, the repository is not cloned again and instead the cache is copied to a temporary folder where the files are checked out from the sparse-checkout.
+* `package config`: Renamed `project config` to `package config` (PR #636). Now that we start using the config more, we came to the conclusion that "package" was better suited than "project".
-* `resources_test`: Add a `resources_test` field to the `_viash.yaml` to specify resources that are needed during testing (PR #709).
- Currently it is up to the user or CI to make sure these resources are available in the `resources_test` directory during testing.
+* `ns exec`: Added an extra field `{name}` to replace `{functionality-name}` (PR #649). No immediate removal of the old field is planned, but it is deprecated.
-## BUG FIXES
+* `BashWrapper`: Added meta-data field `meta_name` as a replacement for `meta_functionality_name` (PR #649). No immediate removal of the old field is planned, but it is deprecated.
-`dependencies`: Fix resolving of dependencies of dependencies (PR #701). The stricter build config was now lacking the necessary information to resolve dependencies of dependencies.
- We added it back as `.build_info.dependencies` in a more structured, anonymized way.
+* `error message`: Improve the error message when using an invalid field in the config (#PR #662). The error message now includes the field names that are not valid if that happens to be the case or otherwise a more general error message.
-`dependencies`: Fix the `name` field of repositories possibly being outputted in the build config (PR #703).
+* `config mods`: Improve the displayed error message when a config mod could not be applied because of an invalid path (PR #672).
-`symlinks`: Allow following of symlinks when finding configs (PR #704). This improves symlink functionality for `viash ns ...` and dependency resolving.
+* `docker_engine`: Deprecate `registry`, `organization` and `tag` fields in the `docker_engine` (PR #712). Currently these are hardly ever used and instead the `image` field is used to specify the full image name.
-# Viash 0.9.0-RC3 (2024-04-26): Various bug fixes and minor improvements
+* `docker_engine`: Add `target_package` field to the `docker_engine` (PR #712). This field, together with the `target_organization` is used to specify the full built container image name. The fields use proper fallback for the values set in the component config and package config.
-Mainly fixes for code changes from previous release candidates. Some additional minor fixes and QoL improvements are included.
+* `organization`: Remove the `organization` field from the component config (PR #712). The value is now directly used by the `docker_engine` as a fallback from the `target_organization` field.
-## BREAKING CHANGES
+* `ExecutableRunner`: Add parameter `docker_automount_prefix` to allow for a custom prefix for automounted folders (PR #739).
-* `computational requirements`: Use 1000-base units instead of 1024-base units for memory (PR #686). Additionally, the memory units `kib`, `mib`, `gib`, `tib`, and `pib` are added to support 1024-base definitions.
+* `ExecutableRunner`: Make Docker runtime arguments configurable via the `---docker_run_args` argument (PR #740).
-## MINOR CHANGES
+* `export json_schema`: Add `arguments` field to the `Config` schema (PR #755). Only for the non-strict version, the strict version of the viash config has these values merged into `argument_groups`.
-* `error message`: Improve the error message when using an invalid field in the config (#PR #662). The error message now includes the field names that are not valid if that happens to be the case or otherwise a more general error message.
+* `scala`: Update Scala to 2.13.14 (PR #764).
-* `config mods`: Improve the displayed error message when a config mod could not be applied because of an invalid path (PR #672).
+* `NextflowEngine`: Also parse `${id}` and `${key}` aside from `$id` and `$key` as identifier placeholders for filenames (PR #756).
## BUG FIXES
+* `__merge__`: Handle invalid yaml during merging (PR #570). There was not enough error handling during this operation. Switched to the more advanced `Convert.textToJson` helper method.
+
+* `config`: Anonymize paths in the config when outputting the config (PR #625).
+
+* `schema`: Don't require undocumented fields to set default values and add the `links` and `reference` fields to functionality as they were not meant only to be in the project config (PR #636).
+
* `export json_schema`: Fix minor inconsistencies and make the strict schema stricter by adapting to what Viash will effectively return (PR #666).
* `deprecation & removal warning`: Improve the displayed warning where a deprecated or removed field could display a double '.' when it field was located at the root level (PR #671).
@@ -129,80 +146,42 @@ Mainly fixes for code changes from previous release candidates. Some additional
* `runners & engines`: When applying a filter on empty runners or engines, the fallback default `native engine` and `executable runner` respectively are set before applying the filter (PR #691).
-# Viash 0.9.0-RC2 (2024-02-23): Restructure the config and change some default values
-
-The `.functionality` layer has been removed from the config and all fields have been moved to the top layer.
-The default `multiple_sep` has been changed from `:` to `;` to avoid conflicts with paths like `s3://foo/bar`.
-
-## BREAKING CHANGES
-
-* `arguments`: Change default `multiple_sep` from `:` to `;` to avoid conflicts with paths like `s3://foo/bar` (PR #645).
- The previous behaviour of using `multiple_sep: ":"` can be achieved by adding a config mod to the `_viash.yaml`:
- ```yaml
- config_mods: |
- .functionality.argument_groups[true].arguments[.multiple == true].multiple_sep := ":"
- ```
-
-* `functionality`: Remove the `functionality` layer from the config and move all fields to the top layer (PR #649).
-
-## MINOR CHANGES
-
-* `package config`: Renamed `project config` to `package config` (PR #636). Now that we start using the config more, we came to the conclusion that "package" was better suited that "project".
-
-* `ns exec`: Added an extra field `{name}` to replace `{functionality-name}` (PR #649). No immediate removal of the old field is planned, but it is deprecated.
-
-* `BashWrapper`: Added meta-data field `meta_name` as a replacement for `meta_functionality_name` (PR #649). No immediate removal of the old field is planned, but it is deprecated.
-
-## BUG FIXES
-
-* `schema`: Don't require undocumented fields to set default values and add the `links` and `reference` fields to functionality as they were not meant only to be in the project config (PR #636).
-
-# Viash 0.9.0-RC1 (2024-01-26): Restructure platforms into runners and engines
-
-This release restructures the `platforms` field into `runners` and `engines`.
-Additionally changes are made to sanitize the built config output and include additional relevant meta data.
-
-## BREAKING CHANGES
-
-* `runners` and `engines`: The usage of `platforms` is deprecated and instead these are split into `runners` and `engines` (PR #510).
- The `platforms` field is still supported but will be removed in a future release.
- In brief, the `native platform` became a `native engine` and `docker platform` became a `docker engine`.
- Additionally, the `native platform` and `docker platform` became a `executable runner`, `nextflow platform` became a `nextflow runner`.
- The fields of `docker platform` is split between `docker engine` and `docker runner`: `port`, `workdir`, `setup_strategy`, and `run_args` (set to `docker_run_args`) are captured by the `runner` as they define how the component is run. The other fields are captured by the `engine` as they define the environment in which the component is run. One exception is `chown` which is rarely set to false and is now always enabled.
-
-* `arguments`: Merge arguments into argument_groups during a json decode prepare step (PR #574). The `--parse_argument_groups` option from `ns list` and `config view` is deprecated as it is now always enabled.
+* `dependencies`: Fix resolving of dependencies of dependencies (PR #701). The stricter build config was now lacking the necessary information to resolve dependencies of dependencies.
+ We added it back as `.build_info.dependencies` in a more structured, anonymized way.
-## NEW FUNCTIONALITY
+* `dependencies`: Fix the `name` field of repositories possibly being outputted in the build config (PR #703).
-* `export json_schema`: Add a `--strict` option to output a subset of the schema representing the internal structure of the Viash config (PR #564).
-
-* `config view` and `ns list`: Do not output internal functionality fields (#564). Additionally, add a validation that no internal fields are present when reading a Viash config file.
+* `symlinks`: Allow following of symlinks when finding configs (PR #704). This improves symlink functionality for `viash ns ...` and dependency resolving.
-* `project config`: Add fields in the project config to specify default values for component config fields (PR #612). This allows for a more DRY approach to defining the same values for multiple components.
+* `build_info`: Correctly set the `.build_info.executable` to `main.nf` when building a component with a Nextflow runner (PR #720).
-## MINOR CHANGES
+* `vsh organization`: ViashHub repositories now use `vsh` as the default organization (PR #718).
+ Instead of having to specify `repo: vsh/repo_name`, you can now just specify `repo: repo_name`, which is now also the prefered way.
-* `testbenches`: Add testbenches for local dependencies (PR #565).
+* `testbenches`: Add a testbench to verify dependencies in dependencies from scratch (PR #721).
+ The components are built from scratch and the dependencies are resolved from the local repositories.
-* `testbenches`: Refactor testbenches helper functions to uniformize them (PR #565).
+* `docker_engine`: Fix a bug in how the namespace separator is handled (PR #722).
-* `logging`: Preserve log order of StdOut and StdErr messages during reading configs in namespaces (PR #571).
+* `platforms`: Re-introduce the `--platform` and `--apply_platform` arguments to improve backwards compatibility (PR #725).
+ When the argument is used, a deprecation warning message is printed on stderr.
+ Cannot be used together with `--engine` or `--runner` and/or `--apply_engine` or `--apply_runner`.
-* `Java 21 support`: Update Scala to 2.13.12 and update dependencies (PR #602).
+* `nextflow_runner`: Fix refactoring error in the `findStates()` helper function (PR #733).
-* `project config`: Output the project config under the default name `ProjectConfig` instead of `Project` during schema export (PR #631). This is now important as the project config is now part of the component config. Previously this was overridden as the class name was `ViashProject` which was less descriptive.
+* `viash ns exec`: Fix "relative fields" outputting absolute paths (PR# 737). Additionally, improve path resolution when using the `--src` argument.
-## BUG FIXES
+* `viash ns`: Fix viash tripping over its toes when it encounters multiple failed configs (PR #761). A dummy config was used as a placeholder, but it always used the name `failed`, so duplicate config names were generated, which we check for nowadays.
-* `__merge__`: Handle invalid yaml during merging (PR #570). There was not enough error handling during this operation. Switched to the more advanced `Convert.textToJson` helper method.
+* `bashwrapper`: Fix an issue where running `viash test` which builds the test docker container would ignore test failures but subsequential runs would work correctly (PR #754).
-* `config`: Anonymize paths in the config when outputting the config (PR #625).
+* `NextflowEngine`: Fix escaping of odd filename containing special characters (PR #756). Filenames containing a `$` character caused Bash to try to interpret it as a variable.
-# Viash 0.8.7 (yyyy-MM-dd): TODO Add title
+* `json schema`: Fix repositories types with name incorrectly adding `withname` as type (PR #768).
-## BUG FIXES
+* `json schema`: Change the '$schema' field to 'http://' instead of 'https://' (PR #768). (Some?) Json validators use this value as a token and not as a URL.
-* `viash build`: Fix error handling of non-generic errors in the build process or while pushing docker containers (PR #696).
+* `viash test`: Fix an issue where the tests would not copy package config settings to determine the docker image name (PR #767).
# Viash 0.8.6 (2024-04-26): Bug fixes and improvements for CI
diff --git a/build.sbt b/build.sbt
index d1a66c267..5f2dcf08e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,32 +1,37 @@
name := "viash"
-version := "0.9.0-dev"
+version := "0.9.1-dev"
-scalaVersion := "2.13.12"
+scalaVersion := "3.3.4"
libraryDependencies ++= Seq(
"org.scalactic" %% "scalactic" % "3.2.15" % "test",
"org.scalatest" %% "scalatest" % "3.2.15" % "test",
"org.rogach" %% "scallop" % "5.0.0",
- "org.scala-lang" % "scala-reflect" % scalaVersion.value,
"org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1",
"org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4",
- "com.github.julien-truffaut" %% "monocle-core" % "2.1.0",
- "com.github.julien-truffaut" %% "monocle-macro" % "2.1.0"
+ "dev.optics" %% "monocle-core" % "3.1.0",
+ "dev.optics" %% "monocle-macro" % "3.1.0"
)
-val circeVersion = "0.14.1"
+val circeVersion = "0.14.7"
libraryDependencies ++= Seq(
"io.circe" %% "circe-core",
"io.circe" %% "circe-generic",
"io.circe" %% "circe-parser",
- "io.circe" %% "circe-generic-extras",
- "io.circe" %% "circe-optics",
- "io.circe" %% "circe-yaml"
+ // "io.circe" %% "circe-generic-extras",
+ // "io.circe" %% "circe-optics",
+ // "io.circe" %% "circe-yaml"
).map(_ % circeVersion)
-scalacOptions ++= Seq("-unchecked", "-deprecation")
+libraryDependencies ++= Seq(
+ "io.circe" %% "circe-optics" % "0.15.0",
+ "io.circe" %% "circe-yaml" % "0.15.2",
+)
+
+scalacOptions ++= Seq("-unchecked", "-deprecation", "-explain")
+scalacOptions ++= Seq("-Xmax-inlines", "50")
organization := "Data Intuitive"
startYear := Some(2020)
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 92c2107ef..ebefee87c 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,3 +1,3 @@
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1")
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.9")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.1.0")
addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.9.0")
diff --git a/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf b/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf
index 95b234342..a9b31b711 100644
--- a/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf
+++ b/src/main/resources/io/viash/runners/nextflow/VDSL3Helper.nf
@@ -71,7 +71,11 @@ def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) {
val = val.join(par.multiple_sep)
}
if (par.direction == "output" && par.type == "file") {
- val = val.replaceAll('\\$id', id).replaceAll('\\$key', key)
+ val = val
+ .replaceAll('\\$id', id)
+ .replaceAll('\\$\\{id\\}', id)
+ .replaceAll('\\$key', key)
+ .replaceAll('\\$\\{key\\}', key)
}
[parName, val]
}
@@ -202,7 +206,8 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
def createParentStr = meta.config.allArguments
.findAll { it.type == "file" && it.direction == "output" && it.create_parent }
.collect { par ->
- "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }"
+ def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]"
+ "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }"
}
.join("\n")
@@ -210,8 +215,8 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
def inputFileExports = meta.config.allArguments
.findAll { it.type == "file" && it.direction.toLowerCase() == "input" }
.collect { par ->
- def viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})"
- "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}=\\\"\" + ${viash_par_contents} + \"\\\"\"}"
+ def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}"
+ "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}"
}
// NOTE: if using docker, use /tmp instead of tmpDir!
@@ -248,6 +253,7 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
def procStr =
"""nextflow.enable.dsl=2
|
+ |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") }
|process $procKey {$drctvStrs
|input:
| tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources")
@@ -259,10 +265,9 @@ def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) {
|$stub
|\"\"\"
|script:$assertStr
- |def escapeText = { s -> s.toString().replaceAll('([`"])', '\\\\\\\\\$1') }
|def parInject = args
| .findAll{key, value -> value != null}
- | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}=\\\"\${escapeText(value)}\\\""}
+ | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"}
| .join("\\n")
|\"\"\"
|# meta exports
diff --git a/src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf b/src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf
index c6a2803c7..67be0aff2 100644
--- a/src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf
+++ b/src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf
@@ -1,7 +1,7 @@
Map _processInputValues(Map inputs, Map config, String id, String key) {
if (!workflow.stubRun) {
config.allArguments.each { arg ->
- if (arg.required) {
+ if (arg.required && arg.direction == "input") {
assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null :
"Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing"
}
diff --git a/src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf b/src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf
index 01eb4ca6d..aded1e8e8 100644
--- a/src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf
+++ b/src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf
@@ -1,12 +1,5 @@
-Map _processOutputValues(Map outputs, Map config, String id, String key) {
+Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) {
if (!workflow.stubRun) {
- config.allArguments.each { arg ->
- if (arg.direction == "output" && arg.required) {
- assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null :
- "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
- }
- }
-
outputs = outputs.collectEntries { name, value ->
def par = config.allArguments.find { it.plainName == name && it.direction == "output" }
assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
@@ -18,3 +11,14 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
}
return outputs
}
+
+void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) {
+ if (!workflow.stubRun) {
+ config.allArguments.each { arg ->
+ if (arg.direction == "output" && arg.required) {
+ assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null :
+ "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing"
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf b/src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf
new file mode 100644
index 000000000..a0270d168
--- /dev/null
+++ b/src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf
@@ -0,0 +1,154 @@
+def publishFiles(Map args) {
+ def key_ = args.get("key")
+
+ assert key_ != null : "publishFiles: key must be specified"
+
+ workflow publishFilesWf {
+ take: input_ch
+ main:
+ input_ch
+ | map { tup ->
+ def id_ = tup[0]
+ def state_ = tup[1]
+
+ // the input files and the target output filenames
+ def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
+ def inputFiles_ = inputoutputFilenames_[0]
+ def outputFilenames_ = inputoutputFilenames_[1]
+
+ [id_, inputFiles_, outputFilenames_]
+ }
+ | publishFilesProc
+ emit: input_ch
+ }
+ return publishFilesWf
+}
+
+process publishFilesProc {
+ // todo: check publishpath?
+ publishDir path: "${getPublishDir()}/", mode: "copy"
+ tag "$id"
+ input:
+ tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+ output:
+ tuple val(id), path{outputFiles}
+ script:
+ def copyCommands = [
+ inputFiles instanceof List ? inputFiles : [inputFiles],
+ outputFiles instanceof List ? outputFiles : [outputFiles]
+ ]
+ .transpose()
+ .collectMany{infile, outfile ->
+ if (infile.toString() != outfile.toString()) {
+ [
+ "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
+ "cp -r '${infile.toString()}' '${outfile.toString()}'"
+ ]
+ } else {
+ // no need to copy if infile is the same as outfile
+ []
+ }
+ }
+ """
+ echo "Copying output files to destination folder"
+ ${copyCommands.join("\n ")}
+ """
+}
+
+
+// this assumes that the state contains no other values other than those specified in the config
+def publishFilesByConfig(Map args) {
+ def config = args.get("config")
+ assert config != null : "publishFilesByConfig: config must be specified"
+
+ def key_ = args.get("key", config.name)
+ assert key_ != null : "publishFilesByConfig: key must be specified"
+
+ workflow publishFilesSimpleWf {
+ take: input_ch
+ main:
+ input_ch
+ | map { tup ->
+ def id_ = tup[0]
+ def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10]
+ def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad']
+
+
+ // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+ // - key is a String
+ // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
+ // - inputPath is a List[Path]
+ // - outputFilename is a List[String]
+ // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
+ def processedState =
+ config.allArguments
+ .findAll { it.direction == "output" }
+ .collectMany { par ->
+ def plainName_ = par.plainName
+ // if the state does not contain the key, it's an
+ // optional argument for which the component did
+ // not generate any output OR multiple channels were emitted
+ // and the output was just not added to using the channel
+ // that is now being parsed
+ if (!state_.containsKey(plainName_)) {
+ return []
+ }
+ def value = state_[plainName_]
+ // if the parameter is not a file, it should be stored
+ // in the state as-is, but is not something that needs
+ // to be copied from the source path to the dest path
+ if (par.type != "file") {
+ return [[inputPath: [], outputFilename: []]]
+ }
+ // if the orig state does not contain this filename,
+ // it's an optional argument for which the user specified
+ // that it should not be returned as a state
+ if (!origState_.containsKey(plainName_)) {
+ return []
+ }
+ def filenameTemplate = origState_[plainName_]
+ // if the pararameter is multiple: true, fetch the template
+ if (par.multiple && filenameTemplate instanceof List) {
+ filenameTemplate = filenameTemplate[0]
+ }
+ // instantiate the template
+ def filename = filenameTemplate
+ .replaceAll('\\$id', id_)
+ .replaceAll('\\$\\{id\\}', id_)
+ .replaceAll('\\$key', key_)
+ .replaceAll('\\$\\{key\\}', key_)
+ if (par.multiple) {
+ // if the parameter is multiple: true, the filename
+ // should contain a wildcard '*' that is replaced with
+ // the index of the file
+ assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}"
+ def outputPerFile = value.withIndex().collect{ val, ix ->
+ def filename_ix = filename.replace("*", ix.toString())
+ def inputPath = val instanceof File ? val.toPath() : val
+ [inputPath: inputPath, outputFilename: filename_ix]
+ }
+ def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key ->
+ [key, outputPerFile.collect{dic -> dic[key]}]
+ }
+ return [[key: plainName_] + transposedOutputs]
+ } else {
+ def value_ = java.nio.file.Paths.get(filename)
+ def inputPath = value instanceof File ? value.toPath() : value
+ return [[inputPath: [inputPath], outputFilename: [filename]]]
+ }
+ }
+
+ def inputPaths = processedState.collectMany{it.inputPath}
+ def outputFilenames = processedState.collectMany{it.outputFilename}
+
+
+ [id_, inputPaths, outputFilenames]
+ }
+ | publishFilesProc
+ emit: input_ch
+ }
+ return publishFilesSimpleWf
+}
+
+
+
diff --git a/src/main/resources/io/viash/runners/nextflow/states/publishStates.nf b/src/main/resources/io/viash/runners/nextflow/states/publishStates.nf
index b35369a63..c57fbdfea 100644
--- a/src/main/resources/io/viash/runners/nextflow/states/publishStates.nf
+++ b/src/main/resources/io/viash/runners/nextflow/states/publishStates.nf
@@ -54,19 +54,19 @@ def publishStates(Map args) {
// the input files and the target output filenames
def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose()
- def inputFiles_ = inputoutputFilenames_[0]
- def outputFilenames_ = inputoutputFilenames_[1]
def yamlFilename = yamlTemplate_
.replaceAll('\\$id', id_)
+ .replaceAll('\\$\\{id\\}', id_)
.replaceAll('\\$key', key_)
+ .replaceAll('\\$\\{key\\}', key_)
// TODO: do the pathnames in state_ match up with the outputFilenames_?
// convert state to yaml blob
def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename))
- [id_, yamlBlob_, yamlFilename, inputFiles_, outputFilenames_]
+ [id_, yamlBlob_, yamlFilename]
}
| publishStatesProc
emit: input_ch
@@ -78,33 +78,17 @@ process publishStatesProc {
publishDir path: "${getPublishDir()}/", mode: "copy"
tag "$id"
input:
- tuple val(id), val(yamlBlob), val(yamlFile), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles)
+ tuple val(id), val(yamlBlob), val(yamlFile)
output:
- tuple val(id), path{[yamlFile] + outputFiles}
+ tuple val(id), path{[yamlFile]}
script:
- def copyCommands = [
- inputFiles instanceof List ? inputFiles : [inputFiles],
- outputFiles instanceof List ? outputFiles : [outputFiles]
- ]
- .transpose()
- .collectMany{infile, outfile ->
- if (infile.toString() != outfile.toString()) {
- [
- "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"",
- "cp -r '${infile.toString()}' '${outfile.toString()}'"
- ]
- } else {
- // no need to copy if infile is the same as outfile
- []
- }
- }
"""
-mkdir -p "\$(dirname '${yamlFile}')"
-echo "Storing state as yaml"
-echo '${yamlBlob}' > '${yamlFile}'
-echo "Copying output files to destination folder"
-${copyCommands.join("\n ")}
-"""
+ mkdir -p "\$(dirname '${yamlFile}')"
+ echo "Storing state as yaml"
+ cat > '${yamlFile}' << HERE
+${yamlBlob}
+HERE
+ """
}
@@ -130,16 +114,15 @@ def publishStatesByConfig(Map args) {
def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml'
def yamlFilename = yamlTemplate
.replaceAll('\\$id', id_)
+ .replaceAll('\\$\\{id\\}', id_)
.replaceAll('\\$key', key_)
+ .replaceAll('\\$\\{key\\}', key_)
def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent()
- // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where
+ // the processed state is a list of [key, value] tuples, where
// - key is a String
// - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path)
- // - inputPath is a List[Path]
- // - outputFilename is a List[String]
// - (key, value) are the tuples that will be saved to the state.yaml file
- // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml)
def processedState =
config.allArguments
.findAll { it.direction == "output" }
@@ -156,7 +139,7 @@ def publishStatesByConfig(Map args) {
// in the state as-is, but is not something that needs
// to be copied from the source path to the dest path
if (par.type != "file") {
- return [[key: plainName_, value: value, inputPath: [], outputFilename: []]]
+ return [[key: plainName_, value: value]]
}
// if the orig state does not contain this filename,
// it's an optional argument for which the user specified
@@ -172,7 +155,9 @@ def publishStatesByConfig(Map args) {
// instantiate the template
def filename = filenameTemplate
.replaceAll('\\$id', id_)
+ .replaceAll('\\$\\{id\\}', id_)
.replaceAll('\\$key', key_)
+ .replaceAll('\\$\\{key\\}', key_)
if (par.multiple) {
// if the parameter is multiple: true, the filename
// should contain a wildcard '*' that is replaced with
@@ -185,13 +170,9 @@ def publishStatesByConfig(Map args) {
if (yamlDir != null) {
value_ = yamlDir.relativize(value_)
}
- def inputPath = val instanceof File ? val.toPath() : val
- [value: value_, inputPath: inputPath, outputFilename: filename_ix]
- }
- def transposedOutputs = ["value", "inputPath", "outputFilename"].collectEntries{ key ->
- [key, outputPerFile.collect{dic -> dic[key]}]
+ return value_
}
- return [[key: plainName_] + transposedOutputs]
+ return [["key": plainName_, "value": outputPerFile]]
} else {
def value_ = java.nio.file.Paths.get(filename)
// if id contains a slash
@@ -199,18 +180,17 @@ def publishStatesByConfig(Map args) {
value_ = yamlDir.relativize(value_)
}
def inputPath = value instanceof File ? value.toPath() : value
- return [[key: plainName_, value: value_, inputPath: [inputPath], outputFilename: [filename]]]
+ return [["key": plainName_, value: value_]]
}
}
+
def updatedState_ = processedState.collectEntries{[it.key, it.value]}
- def inputPaths = processedState.collectMany{it.inputPath}
- def outputFilenames = processedState.collectMany{it.outputFilename}
// convert state to yaml blob
def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_)
- [id_, yamlBlob_, yamlFilename, inputPaths, outputFilenames]
+ [id_, yamlBlob_, yamlFilename]
}
| publishStatesProc
emit: input_ch
diff --git a/src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf b/src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf
index 50d78b4c9..2361cfa0a 100644
--- a/src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf
+++ b/src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf
@@ -10,7 +10,8 @@ def _debug(workflowArgs, debugKey) {
def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta)
def key_ = workflowArgs["key"]
-
+ def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName}
+
workflow workflowInstance {
take: input_
@@ -167,12 +168,36 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
}
// TODO: move some of the _meta.join_id wrangling to the safeJoin() function.
- def chInitialOutput = chArgsWithDefaults
+ def chInitialOutputMulti = chArgsWithDefaults
| _debug(workflowArgs, "processed")
// run workflow
| innerWorkflowFactory(workflowArgs)
- // check output tuple
- | map { id_, output_ ->
+ def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti]
+ assert chInitialOutputList.size() > 0: "should have emitted at least one output channel"
+ // Add a channel ID to the events, which designates the channel the event was emitted from as a running number
+ // This number is used to sort the events later when the events are gathered from across the channels.
+ def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex ->
+ def newChannel = channel
+ | map {tuple ->
+ assert tuple instanceof List :
+ "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" +
+ " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" +
+ " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}"
+
+ def newEvent = [channelIndex] + tuple
+ return newEvent
+ }
+ return newChannel
+ }
+ // Put the events into 1 channel, cover case where there is only one channel is emitted
+ def chInitialOutput = chInitialOutputList.size() > 1 ? \
+ chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \
+ chInitialOutputListWithIndexedEvents[0]
+ def chInitialOutputProcessed = chInitialOutput
+ | map { tuple ->
+ def channelId = tuple[0]
+ def id_ = tuple[1]
+ def output_ = tuple[2]
// see if output map contains metadata
def meta_ =
@@ -185,19 +210,95 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
output_ = output_.findAll{k, v -> k != "_meta"}
// check value types
- output_ = _processOutputValues(output_, meta.config, id_, key_)
+ output_ = _checkValidOutputArgument(output_, meta.config, id_, key_)
- // simplify output if need be
- if (workflowArgs.auto.simplifyOutput && output_.size() == 1) {
- output_ = output_.values()[0]
+ [join_id, channelId, id_, output_]
+ }
+ // | view{"chInitialOutput: ${it.take(3)}"}
+
+ // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...]
+ def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_)
+ // input tuple format: [join_id, channel_id, id, output, prev_state, ...]
+ // output tuple format: [join_id, channel_id, id, new_state, ...]
+ | map{ tup ->
+ def new_state = workflowArgs.toState(tup.drop(2).take(3))
+ tup.take(3) + [new_state] + tup.drop(5)
+ }
+ if (workflowArgs.auto.publish == "state") {
+ def chPublishFiles = chPublishWithPreviousState
+ // input tuple format: [join_id, channel_id, id, new_state, ...]
+ // output tuple format: [join_id, channel_id, id, new_state]
+ | map{ tup ->
+ tup.take(4)
}
- [join_id, id_, output_]
+ safeJoin(chPublishFiles, chArgsWithDefaults, key_)
+ // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...]
+ // output tuple format: [id, new_state, orig_state]
+ | map { tup ->
+ tup.drop(2).take(3)
+ }
+ | publishFilesByConfig(key: key_, config: meta.config)
+ }
+ // Join the state from the events that were emitted from different channels
+ def chJoined = chInitialOutputProcessed
+ | map {tuple ->
+ def join_id = tuple[0]
+ def channel_id = tuple[1]
+ def id = tuple[2]
+ def other = tuple.drop(3)
+ // Below, groupTuple is used to join the events. To make sure resuming a workflow
+ // keeps working, the output state must be deterministic. This means the state needs to be
+ // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash',
+ // but hashing the state when it is large can be problematic in terms of performance.
+ // Therefore, a custom comparator function is provided. We add the channel ID to the
+ // states so that we can use the channel ID to sort the items.
+ def stateWithChannelID = [[channel_id] * other.size(), other].transpose()
+ // A comparator that is provided to groupTuple's 'sort' argument is applied
+ // to all elements of the event tuple (that is not the 'id'). The comparator
+ // closure that is used below expects the input to be List. So the join_id and
+ // channel_id must also be wrapped in a list.
+ [[join_id], [channel_id], id] + stateWithChannelID
}
- // | view{"chInitialOutput: ${it.take(3)}"}
+ | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true)
+ | map {join_ids, _, id, statesWithChannelID ->
+ // Remove the channel IDs from the states
+ def states = statesWithChannelID.collect{it[1]}
+ def newJoinId = join_ids.flatten().unique{a, b -> a <=> b}
+ assert newJoinId.size() == 1: "Multiple events were emitted for '$id'."
+ def newJoinIdUnique = newJoinId[0]
+ def newState = states.inject([:]){ old_state, state_to_add ->
+ def stateToAddNoMultiple = state_to_add.findAll{k, v -> !multipleArgs.contains(k)}
+ // First add non multiple arguments
+
+ def overlap = old_state.keySet().intersect(stateToAddNoMultiple.keySet())
+ assert overlap.isEmpty() : "ID $id: multiple entries for " +
+ " argument(s) $overlap were emitted."
+ def return_state = old_state + stateToAddNoMultiple
+
+ // Add `multiple: true` arguments
+ def stateToAddMultiple = state_to_add.findAll{k, v -> multipleArgs.contains(k)}
+ stateToAddMultiple.each {k, v ->
+ def currentKey = return_state.getOrDefault(k, [])
+ def currentKeyList = currentKey instanceof List ? currentKey : [currentKey]
+ currentKeyList.add(v)
+ return_state[k] = currentKeyList
+ }
+ return return_state
+ }
+
+ _checkAllRequiredOuputsPresent(newState, meta.config, id, key_)
+ // simplify output if need be
+ if (workflowArgs.auto.simplifyOutput && newState.size() == 1) {
+ newState = newState.values()[0]
+ }
+
+ return [newJoinIdUnique, id, newState]
+ }
+
// join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...]
- def chNewState = safeJoin(chInitialOutput, chRunFiltered, key_)
+ def chNewState = safeJoin(chJoined, chRunFiltered, key_)
// input tuple format: [join_id, id, output, prev_state, ...]
// output tuple format: [join_id, id, new_state, ...]
| map{ tup ->
@@ -206,23 +307,21 @@ def workflowFactory(Map args, Map defaultWfArgs, Map meta) {
}
if (workflowArgs.auto.publish == "state") {
- def chPublish = chNewState
+ def chPublishStates = chNewState
// input tuple format: [join_id, id, new_state, ...]
// output tuple format: [join_id, id, new_state]
| map{ tup ->
tup.take(3)
}
- safeJoin(chPublish, chArgsWithDefaults, key_)
+ safeJoin(chPublishStates, chArgsWithDefaults, key_)
// input tuple format: [join_id, id, new_state, orig_state, ...]
// output tuple format: [id, new_state, orig_state]
| map { tup ->
tup.drop(1).take(3)
- }
+ }
| publishStatesByConfig(key: key_, config: meta.config)
}
-
- // remove join_id and meta
chReturn = chNewState
| map { tup ->
// input tuple format: [join_id, id, new_state, ...]
diff --git a/src/main/scala/io/viash/Main.scala b/src/main/scala/io/viash/Main.scala
index 7ecbac816..79909f47f 100644
--- a/src/main/scala/io/viash/Main.scala
+++ b/src/main/scala/io/viash/Main.scala
@@ -34,6 +34,7 @@ import io.viash.helpers.LoggerLevel
import io.viash.runners.Runner
import io.viash.config.AppliedConfig
import io.viash.engines.Engine
+import io.viash.helpers.data_structures.*
object Main extends Logging {
private val pkg = getClass.getPackage
@@ -551,7 +552,7 @@ object Main extends Logging {
configs0
}
- configs1
+ configs1
}
// Handle dependencies operations for a single config
diff --git a/src/main/scala/io/viash/ViashBuild.scala b/src/main/scala/io/viash/ViashBuild.scala
index 72b7aa032..64f277f05 100644
--- a/src/main/scala/io/viash/ViashBuild.scala
+++ b/src/main/scala/io/viash/ViashBuild.scala
@@ -19,6 +19,7 @@ package io.viash
import java.nio.file.{Files, Paths}
import scala.sys.process.{Process, ProcessLogger}
+import io.viash.helpers.status
import io.viash.helpers.status._
import config._
@@ -31,7 +32,7 @@ object ViashBuild extends Logging {
output: String,
setup: Option[String] = None,
push: Boolean = false
- ): Status = {
+ ): status.Status = {
val resources = appliedConfig.generateRunner(false)
// create dir
diff --git a/src/main/scala/io/viash/ViashConfig.scala b/src/main/scala/io/viash/ViashConfig.scala
index fb1f5cea9..fc79409a1 100644
--- a/src/main/scala/io/viash/ViashConfig.scala
+++ b/src/main/scala/io/viash/ViashConfig.scala
@@ -49,7 +49,7 @@ object ViashConfig extends Logging{
throw new ExitException(1)
}
// check if we can read code
- if (config.mainScript.get.read.isEmpty) {
+ if (config.mainScript.get.readSome.isEmpty) {
infoOut("Could not read main script in the Viash config.")
throw new ExitException(1)
}
diff --git a/src/main/scala/io/viash/ViashNamespace.scala b/src/main/scala/io/viash/ViashNamespace.scala
index 57a1b8441..6b85167ac 100644
--- a/src/main/scala/io/viash/ViashNamespace.scala
+++ b/src/main/scala/io/viash/ViashNamespace.scala
@@ -33,6 +33,7 @@ import io.viash.helpers.LoggerOutput
import io.viash.helpers.LoggerLevel
import io.viash.runners.Runner
import io.viash.config.AppliedConfig
+import io.viash.config.{ScopeEnum, Scope}
object ViashNamespace extends Logging {
@@ -52,12 +53,18 @@ object ViashNamespace extends Logging {
list.foreach(f)
}
- def targetOutputPath(targetDir: String, runnerId: String, config: Config): String =
- targetOutputPath(targetDir, runnerId, config.namespace, config.name)
+ def targetOutputPath(targetDir: String, runnerId: String, config: Config): String = {
+ val scope = config.scope match {
+ case Left(value) => Scope(value)
+ case Right(value) => value
+ }
+ targetOutputPath(targetDir, runnerId, scope.target, config.namespace, config.name)
+ }
def targetOutputPath(
targetDir: String,
runnerId: String,
+ scope: ScopeEnum,
namespace: Option[String],
name: String
): String = {
@@ -65,7 +72,12 @@ object ViashNamespace extends Logging {
case Some(ns) => ns + "/"
case None => ""
}
- s"$targetDir/$runnerId/$nsStr$name"
+ val scopeStr = scope match {
+ case ScopeEnum.Test => "_test/"
+ case ScopeEnum.Private => "_private/"
+ case ScopeEnum.Public => ""
+ }
+ s"$targetDir/$scopeStr$runnerId/$nsStr$name"
}
def build(
diff --git a/src/main/scala/io/viash/ViashTest.scala b/src/main/scala/io/viash/ViashTest.scala
index 8c6b9deed..12efb1bc4 100644
--- a/src/main/scala/io/viash/ViashTest.scala
+++ b/src/main/scala/io/viash/ViashTest.scala
@@ -240,7 +240,7 @@ object ViashTest extends Logging {
val tests = conf.test_resources
val testResults = tests.filter(_.isInstanceOf[Script]).map {
- case test: Script if test.read.isEmpty =>
+ case test: Script if test.readSome.isEmpty =>
TestOutput(test.filename, 1, "Test script does not exist.", "", 0L)
case test: Script =>
@@ -272,6 +272,8 @@ object ViashTest extends Logging {
// Make sure we'll be using the same docker registry set in 'links' so we can have the same docker image id.
// Copy the whole case class instead of selective copy.
links = conf.links,
+ // copy configuration for package name, organization
+ package_config = conf.package_config,
))(appliedConfig)
// generate bash script for test
@@ -283,7 +285,7 @@ object ViashTest extends Logging {
val configYaml = ConfigMeta.toMetaFile(appliedConfig, Some(dir))
// assemble full resources list for test
- val confFinal = resourcesLens.set(
+ val confFinal = resourcesLens.replace(
testBash ::
// the executable, wrapped with an executable runner,
// to be run inside of the runner of the test
diff --git a/src/main/scala/io/viash/cli/DocumentedSubcommand.scala b/src/main/scala/io/viash/cli/DocumentedSubcommand.scala
index 70e4ca169..4ba9b2369 100644
--- a/src/main/scala/io/viash/cli/DocumentedSubcommand.scala
+++ b/src/main/scala/io/viash/cli/DocumentedSubcommand.scala
@@ -21,7 +21,7 @@ import org.rogach.scallop.Subcommand
import org.rogach.scallop.ScallopOptionGroup
import org.rogach.scallop.ValueConverter
import org.rogach.scallop.ScallopOption
-import scala.reflect.runtime.universe._
+import io.viash.helpers.typeOf
/**
* Wrapper class for Subcommand to expose protected members
@@ -76,7 +76,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co
// We need to get the TypeTag[A], which changes the interface of 'opt', however since we have default values we can't just overload the methods.
// The same goes for 'trailArgs'. Not really for 'choice' but it's better to keep the same change in naming schema here too.
- def registerOpt[A](
+ inline def registerOpt[A](
name: String,
short: Option[Char] = None,
descr: String = "",
@@ -86,9 +86,8 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co
argName: String = "arg",
hidden: Boolean = false,
group: ScallopOptionGroup = null
- )(implicit conv:ValueConverter[A], tag: TypeTag[A]): ScallopOption[A] = {
+ )(implicit conv:ValueConverter[A]): ScallopOption[A] = {
- val `type` = tag.tpe
val cleanName = name match {
case null => ""
case _ => name
@@ -103,14 +102,14 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co
argName = Some(argName),
hidden = hidden,
choices = None,
- `type` = `type`.toString(),
+ `type` = typeOf[A],
optType = "opt"
)
registeredOpts = registeredOpts :+ registeredOpt
opt(name, short.getOrElse('\u0000'), removeMarkup(descr), default, validate, required, argName, hidden, short.isEmpty, group)
}
- def registerChoice(
+ inline def registerChoice(
choices: Seq[String],
name: String,
short: Option[Char],
@@ -143,7 +142,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co
choice(choices, name, short.getOrElse('\u0000'), removeMarkup(descr), default, required, argName, hidden, short.isEmpty, group)
}
- def registerTrailArg[A](
+ inline def registerTrailArg[A](
name: String,
descr: String = "",
validate: A => Boolean = (_:A) => true,
@@ -151,9 +150,8 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co
default: => Option[A] = None,
hidden: Boolean = false,
group: ScallopOptionGroup = null
- )(implicit conv:ValueConverter[A], tag: TypeTag[A]) = {
+ )(implicit conv:ValueConverter[A]) = {
- val `type` = tag.tpe
val cleanName = name match {
case null => ""
case _ => name
@@ -168,7 +166,7 @@ class DocumentedSubcommand(commandNameAndAliases: String*) extends Subcommand(co
argName = None,
hidden = hidden,
choices = None,
- `type` = `type`.toString,
+ `type` = typeOf[A],
optType = "trailArgs"
)
registeredOpts = registeredOpts :+ registeredOpt
diff --git a/src/main/scala/io/viash/cli/package.scala b/src/main/scala/io/viash/cli/package.scala
index 11dd143a4..586ddc643 100644
--- a/src/main/scala/io/viash/cli/package.scala
+++ b/src/main/scala/io/viash/cli/package.scala
@@ -18,7 +18,6 @@
package io.viash
import io.circe.Encoder
-import io.circe.generic.extras.semiauto.deriveConfiguredEncoder
import org.rogach.scallop.CliOption
package object cli {
diff --git a/src/main/scala/io/viash/config/ArgumentGroup.scala b/src/main/scala/io/viash/config/ArgumentGroup.scala
index a49ad06c3..a311268f4 100644
--- a/src/main/scala/io/viash/config/ArgumentGroup.scala
+++ b/src/main/scala/io/viash/config/ArgumentGroup.scala
@@ -40,7 +40,7 @@ import io.viash.schemas._
| - name: "--output_optional"
| type: file
| direction: output
- |""".stripMargin,
+ |""",
"yaml")
case class ArgumentGroup(
@description("The name of the argument group.")
@@ -62,7 +62,7 @@ case class ArgumentGroup(
@example(
"""description: |
| A (multiline) description of the purpose of the arguments
- | in this argument group.""".stripMargin, "yaml")
+ | in this argument group.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -76,7 +76,7 @@ case class ArgumentGroup(
| - @[boolean](arg_boolean)
| - @[boolean_true](arg_boolean_true)
| - @[boolean_false](arg_boolean_false)
- |""".stripMargin)
+ |""")
@example(
"""arguments:
| - name: --foo
@@ -91,7 +91,7 @@ case class ArgumentGroup(
| multiple_sep: ";"
| - name: --bar
| type: string
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
arguments: List[Argument[_]] = Nil
diff --git a/src/main/scala/io/viash/config/Author.scala b/src/main/scala/io/viash/config/Author.scala
index 6a1f20443..b07e9d0a7 100644
--- a/src/main/scala/io/viash/config/Author.scala
+++ b/src/main/scala/io/viash/config/Author.scala
@@ -34,7 +34,7 @@ import io.viash.schemas._
| twitter: janedoe
| orcid: XXAABBCCXX
| groups: [ one, two, three ]
- |""".stripMargin, "yaml")
+ |""", "yaml")
case class Author(
@description("Full name of the author, usually in the name of FirstName MiddleName LastName.")
name: String,
@@ -48,7 +48,7 @@ case class Author(
|* `"author"`: Authors who have made substantial contributions to the component.
|* `"maintainer"`: The maintainer of the component.
|* `"contributor"`: Authors who have made smaller contributions (such as code patches etc.).
- |""".stripMargin)
+ |""")
@default("Empty")
roles: OneOrMore[String] = Nil,
diff --git a/src/main/scala/io/viash/config/ComputationalRequirements.scala b/src/main/scala/io/viash/config/ComputationalRequirements.scala
index 07efb2be2..29cef36ee 100644
--- a/src/main/scala/io/viash/config/ComputationalRequirements.scala
+++ b/src/main/scala/io/viash/config/ComputationalRequirements.scala
@@ -24,7 +24,7 @@ import io.viash.schemas._
"""requirements:
| cpus: 5
| memory: 10GB
- |""".stripMargin,
+ |""",
"yaml")
@since("Viash 0.6.0")
case class ComputationalRequirements(
diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala
index b2248fe83..8ae66edd8 100644
--- a/src/main/scala/io/viash/config/Config.scala
+++ b/src/main/scala/io/viash/config/Config.scala
@@ -47,11 +47,12 @@ import io.viash.lenses.ConfigLenses._
import Status._
import io.viash.wrapper.BashWrapper
import scala.collection.immutable.ListMap
+import io.viash.helpers.data_structures.oneOrMoreToList
@description(
"""A Viash configuration is a YAML file which contains metadata to describe the behaviour and build target(s) of a component.
|We commonly name this file `config.vsh.yaml` in our examples, but you can name it however you choose.
- |""".stripMargin)
+ |""")
@example(
"""name: hello_world
|arguments:
@@ -67,7 +68,7 @@ import scala.collection.immutable.ListMap
|engines:
| - type: docker
| image: "bash:4.0"
- |""".stripMargin, "yaml")
+ |""", "yaml")
case class Config(
@description("Name of the component and the filename of the executable when built with `viash build`.")
@example("name: this_is_my_component", "yaml")
@@ -83,20 +84,20 @@ case class Config(
@description(
"""A list of @[authors](author). An author must at least have a name, but can also have a list of roles, an e-mail address, and a map of custom properties.
- +
- +Suggested values for roles are:
- +
- +| Role | Abbrev. | Description |
- +|------|---------|-------------|
- +| maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. |
- +| author | aut | for persons who have made substantial contributions to the software. |
- +| contributor | ctb| for persons who have made smaller contributions (such as code patches).
- +| datacontributor | dtc | for persons or organisations that contributed data sets for the software
- +| copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body.
- +| funder | fnd | for persons or organizations that furnished financial support for the development of the software
- +
- +The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive.
- +""".stripMargin('+'))
+ |
+ |Suggested values for roles are:
+ |
+ || Role | Abbrev. | Description |
+ ||------|---------|-------------|
+ || maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. |
+ || author | aut | for persons who have made substantial contributions to the software. |
+ || contributor | ctb| for persons who have made smaller contributions (such as code patches).
+ || datacontributor | dtc | for persons or organisations that contributed data sets for the software
+ || copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body.
+ || funder | fnd | for persons or organizations that furnished financial support for the development of the software
+ |
+ |The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive.
+ |""")
@example(
"""authors:
| - name: Jane Doe
@@ -110,7 +111,7 @@ case class Config(
| - name: Tim Farbe
| roles: [author]
| email: tim@far.be
- |""".stripMargin, "yaml")
+ |""", "yaml")
@since("Viash 0.3.1")
@default("Empty")
authors: List[Author] = Nil,
@@ -122,7 +123,7 @@ case class Config(
| - `description: Description of foo`, a description of the argument group. Multiline descriptions are supported.
| - `arguments: [arg1, arg2, ...]`, list of the arguments.
|
- |""".stripMargin)
+ |""")
@example(
"""argument_groups:
| - name: "Input"
@@ -142,7 +143,7 @@ case class Config(
| - name: "--output_optional"
| type: file
| direction: output
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription(
"""component_name
@@ -160,7 +161,7 @@ case class Config(
|
| --optional_output
| type: file
- |""".stripMargin,
+ |""",
"bash",
"This results in the following output when calling the component with the `--help` argument:")
@since("Viash 0.5.14")
@@ -177,14 +178,14 @@ case class Config(
| * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`.
| * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`.
| * is_executable: `true` / `false`, whether the resulting resource file should be made executable.
- |""".stripMargin)
+ |""")
@example(
"""resources:
| - type: r_script
| path: script.R
| - type: file
| path: resource1.txt
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
resources: List[Resource] = Nil,
@@ -205,9 +206,9 @@ case class Config(
@default("Empty")
@example(
"""description: |
- + This component performs function Y and Z.
- + It is possible to make this a multiline string.
- +""".stripMargin('+'),
+ | This component performs function Y and Z.
+ | It is possible to make this a multiline string.
+ |""",
"yaml")
description: Option[String] = None,
@@ -223,7 +224,7 @@ case class Config(
| - type: r_script
| path: tests/test2.R
| - path: resource1.txt
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
test_resources: List[Resource] = Nil,
@@ -232,7 +233,7 @@ case class Config(
@example(
"""info:
| twitter: wizzkid
- | classes: [ one, two, three ]""".stripMargin, "yaml")
+ | classes: [ one, two, three ]""", "yaml")
@since("Viash 0.4.0")
@default("Empty")
info: Json = Json.Null,
@@ -241,17 +242,26 @@ case class Config(
@since("Viash 0.6.0")
@default("Enabled")
status: Status = Status.Enabled,
+
+ @description(
+ """Defines the scope of the component.
+ |`test`: only available during testing; components aren't published.
+ |`private`: only meant for internal use within a workflow or other component.
+ |`public`: core component or workflow meant for general use.""")
+ @since("Viash 0.9.1")
+ @default("public")
+ scope: Either[ScopeEnum, Scope] = Left(ScopeEnum.Public),
@description(
"""@[Computational requirements](computational_requirements) related to running the component.
|`cpus` specifies the maximum number of (logical) cpus a component is allowed to use., whereas
|`memory` specifies the maximum amount of memory a component is allowed to allicate. Memory units must be
- |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""".stripMargin)
+ |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""")
@example(
"""requirements:
| cpus: 5
| memory: 10GB
- |""".stripMargin,
+ |""",
"yaml")
@since("Viash 0.6.0")
@default("Empty")
@@ -265,21 +275,21 @@ case class Config(
| type: github
| uri: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml",
"Full specification of a repository")
@exampleWithDescription(
"""dependencies:
| - name: qc/multiqc
| repository: "github://openpipelines-bio/modules:0.3.0"
- |""".stripMargin,
+ |""",
"yaml",
"Full specification of a repository using sugar syntax")
@exampleWithDescription(
"""dependencies:
| - name: qc/multiqc
| repository: "openpipelines-bio"
- |""".stripMargin,
+ |""",
"yaml",
"Reference to a repository fully specified under 'repositories'")
@default("Empty")
@@ -287,14 +297,14 @@ case class Config(
@description(
"""(Pre-)defines repositories that can be used as repository in dependencies.
- |Allows reusing repository definitions in case it is used in multiple dependencies.""".stripMargin)
+ |Allows reusing repository definitions in case it is used in multiple dependencies.""")
@example(
"""repositories:
| - name: openpipelines-bio
| type: github
| uri: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
repositories: List[RepositoryWithName] = Nil,
@@ -322,7 +332,7 @@ case class Config(
| journal={Baz},
| year={2024}
| }
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
references: References = References(),
@@ -335,7 +345,7 @@ case class Config(
| homepage: "https://viash.io"
| documentation: "https://viash.io/reference/"
| issue_tracker: "https://github.com/viash-io/viash/issues"
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
links: Links = Links(),
@@ -352,7 +362,7 @@ case class Config(
|
| - @[ExecutableRunner](executable_runner)
| - @[NextflowRunner](nextflow_runner)
- |""".stripMargin)
+ |""")
@since("Viash 0.8.0")
@default("Empty")
runners: List[Runner] = Nil,
@@ -361,7 +371,7 @@ case class Config(
|
| - @[NativeEngine](native_engine)
| - @[DockerEngine](docker_engine)
- |""".stripMargin)
+ |""")
@since("Viash 0.8.0")
@default("Empty")
engines: List[Engine] = Nil,
@@ -377,7 +387,7 @@ case class Config(
@description(
"""The @[functionality](functionality) describes the behaviour of the script in terms of arguments and resources.
|By specifying a few restrictions (e.g. mandatory arguments) and adding some descriptions, Viash will automatically generate a stylish command-line interface for you.
- |""".stripMargin)
+ |""")
@deprecated("Functionality level is deprecated, all functionality fields are now located on the top level of the config file.", "0.9.0", "0.10.0")
@default("")
private val functionality: Functionality = Functionality("foo")
@@ -392,7 +402,7 @@ case class Config(
| - @[boolean](arg_boolean)
| - @[boolean_true](arg_boolean_true)
| - @[boolean_false](arg_boolean_false)
- |""".stripMargin)
+ |""")
@default("Empty")
private val arguments: List[Argument[_]] = Nil
@@ -400,7 +410,7 @@ case class Config(
"""Config inheritance by including YAML partials. This is useful for defining common APIs in
|separate files. `__merge__` can be used in any level of the YAML. For example,
|not just in the config but also in the argument_groups or any of the engines.
- |""".stripMargin)
+ |""")
@example("__merge__: ../api/common_interface.yaml", "yaml")
@since("Viash 0.6.3")
private val `__merge__`: Option[File] = None
@@ -411,7 +421,7 @@ case class Config(
| - @[Native](platform_native)
| - @[Docker](platform_docker)
| - @[Nextflow](platform_nextflow)
- |""".stripMargin)
+ |""")
@default("Empty")
@deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0")
private val platforms: List[Platform] = Nil
@@ -552,7 +562,7 @@ case class Config(
case s: Script => Some(s)
case _ => None
}
- def mainCode: Option[String] = mainScript.flatMap(_.read)
+ def mainCode: Option[String] = mainScript.flatMap(_.readSome)
// provide function to use resources.tail but that allows resources to be an empty list
// If mainScript ends up being None because the first resource isn't a script, return the whole list
def additionalResources = resources match {
@@ -697,8 +707,8 @@ object Config extends Logging {
val resources = conf1.resources.map(_.copyWithAbsolutePath(parentURI, packageDir))
val tests = conf1.test_resources.map(_.copyWithAbsolutePath(parentURI, packageDir))
- val conf2a = resourcesLens.set(resources)(conf1)
- val conf2 = testResourcesLens.set(tests)(conf2a)
+ val conf2a = resourcesLens.replace(resources)(conf1)
+ val conf2 = testResourcesLens.replace(tests)(conf2a)
/* CONFIG 3: add info */
// gather git info
@@ -735,16 +745,21 @@ object Config extends Logging {
val conf4 = conf3.copy(
package_config = viashPackage
)
-
- if (!addOptMainScript) {
- return conf4
- }
-
+
/* CONFIG 5: add main script if config is stored inside script */
// add info and additional resources
- val conf5 = resourcesLens.modify(optScript.toList ::: _)(conf4)
+ val conf5 = addOptMainScript match {
+ case true => resourcesLens.modify(optScript.toList ::: _)(conf4)
+ case false => conf4
+ }
+
+ /* CONFIG 6: Finalize Scope */
+ val conf6 = scopeLens.modify {
+ case Left(scope) => Right(Scope(scope))
+ case right => right
+ }(conf5)
- conf5
+ conf6
}
def readConfigs(
diff --git a/src/main/scala/io/viash/config/ConfigMeta.scala b/src/main/scala/io/viash/config/ConfigMeta.scala
index 63a56cedb..38aa24ea8 100644
--- a/src/main/scala/io/viash/config/ConfigMeta.scala
+++ b/src/main/scala/io/viash/config/ConfigMeta.scala
@@ -118,7 +118,7 @@ object ConfigMeta {
val configYamlStr2 = placeholderMap.foldLeft(configYamlStr) {
case (configStr, (res, placeholder)) =>
val IndentRegex = ("( *)text: \"" + placeholder + "\"").r
- val IndentRegex(indent) = IndentRegex.findFirstIn(configStr).getOrElse("")
+ val IndentRegex(indent) = IndentRegex.findFirstIn(configStr).getOrElse("") : @unchecked
configStr.replace(
"\"" + placeholder + "\"",
"|\n" + indent + " " + res.text.get.replace("\n", "\n " + indent) + "\n"
diff --git a/src/main/scala/io/viash/config/Links.scala b/src/main/scala/io/viash/config/Links.scala
index 95cc0fda5..6f3375470 100644
--- a/src/main/scala/io/viash/config/Links.scala
+++ b/src/main/scala/io/viash/config/Links.scala
@@ -27,7 +27,7 @@ import io.viash.schemas._
| homepage: "https://viash.io"
| documentation: "https://viash.io/reference/"
| issue_tracker: "https://github.com/viash-io/viash/issues"
- |""".stripMargin, "yaml")
+ |""", "yaml")
@since("Viash 0.9.0")
case class Links(
@description("Source repository url.")
diff --git a/src/main/scala/io/viash/config/References.scala b/src/main/scala/io/viash/config/References.scala
index c92bd7049..6dbaa7285 100644
--- a/src/main/scala/io/viash/config/References.scala
+++ b/src/main/scala/io/viash/config/References.scala
@@ -18,7 +18,7 @@
package io.viash.config
import io.viash.schemas._
-import io.viash.helpers.data_structures.OneOrMore
+import io.viash.helpers.data_structures.{OneOrMore, listToOneOrMore}
@description("A list of scholarly sources or publications relevant to the tools or analysis defined in the component. This is important for attribution, scientific reproducibility and transparency.")
@example(
@@ -31,7 +31,7 @@ import io.viash.helpers.data_structures.OneOrMore
| journal={Baz},
| year={2024}
| }
- |""".stripMargin, "yaml")
+ |""", "yaml")
@since("Viash 0.9.0")
case class References(
@description("One or multiple DOI reference(s) of the component.")
@@ -48,7 +48,7 @@ case class References(
| journal={Baz},
| year={2024}
| }
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
bibtex: OneOrMore[String] = Nil,
)
diff --git a/src/main/scala/io/viash/config/Scope.scala b/src/main/scala/io/viash/config/Scope.scala
new file mode 100644
index 000000000..4cb971363
--- /dev/null
+++ b/src/main/scala/io/viash/config/Scope.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2020 Data Intuitive
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+package io.viash.config
+
+import io.viash.schemas.description
+
+enum ScopeEnum {
+ case Test, Private, Public
+}
+
+@description(
+ """Defines the scope of the component.
+ |`test`: only available during testing; components aren't published.
+ |`private`: only meant for internal use within a workflow or other component.
+ |`public`: core component or workflow meant for general use.""")
+case class Scope(
+ @description(
+ """test: image is only used during testing and is transient
+ |private: image is published in the registry
+ |public: image is published in the registry""")
+ image: ScopeEnum,
+ @description(
+ """test: target folder is only used during testing and is transient
+ |private: target folder can be published in target/private or target/dependencies/private
+ |public: target is published in target/executable or target/nextflow"""
+ )
+ target: ScopeEnum,
+)
+
+object Scope {
+ def apply(scopeValue: ScopeEnum): Scope = {
+ Scope(scopeValue, scopeValue)
+ }
+}
diff --git a/src/main/scala/io/viash/config/Status.scala b/src/main/scala/io/viash/config/Status.scala
index f256681fc..65d8bb69c 100644
--- a/src/main/scala/io/viash/config/Status.scala
+++ b/src/main/scala/io/viash/config/Status.scala
@@ -17,9 +17,5 @@
package io.viash.config
-object Status extends Enumeration {
- type Status = Value
- val Enabled = Value("enabled")
- val Disabled = Value("disabled")
- val Deprecated = Value("deprecated")
-}
+enum Status:
+ case Enabled, Disabled, Deprecated
diff --git a/src/main/scala/io/viash/config/arguments/Argument.scala b/src/main/scala/io/viash/config/arguments/Argument.scala
index 98a528a00..d95f5bcf5 100644
--- a/src/main/scala/io/viash/config/arguments/Argument.scala
+++ b/src/main/scala/io/viash/config/arguments/Argument.scala
@@ -32,7 +32,7 @@ import java.nio.file.Paths
| - @[boolean](arg_boolean)
| - @[boolean_true](arg_boolean_true)
| - @[boolean_false](arg_boolean_false)
- |""".stripMargin)
+ |""")
@example(
"""arguments:
| - name: --foo
@@ -47,7 +47,7 @@ import java.nio.file.Paths
| multiple_sep: ";"
| - name: --bar
| type: string
- |""".stripMargin,
+ |""",
"yaml")
@subclass("BooleanArgument")
@subclass("BooleanTrueArgument")
@@ -77,7 +77,7 @@ abstract class Argument[Type] {
val dest: String
private val pattern = "^(-*)(.*)$".r
- val pattern(flags, plainName) = name
+ val pattern(flags, plainName) = name : @unchecked
/** Common parameter name for this argument */
val par: String = dest + "_" + plainName
diff --git a/src/main/scala/io/viash/config/arguments/BooleanArgument.scala b/src/main/scala/io/viash/config/arguments/BooleanArgument.scala
index 90152a27a..3f8542773 100644
--- a/src/main/scala/io/viash/config/arguments/BooleanArgument.scala
+++ b/src/main/scala/io/viash/config/arguments/BooleanArgument.scala
@@ -33,7 +33,7 @@ abstract class BooleanArgumentBase extends Argument[Boolean] {
| default: true
| description: Trim whitespace from the final output
| alternatives: ["-t"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("boolean")
case class BooleanArgument(
@@ -43,7 +43,7 @@ case class BooleanArgument(
| - `--trim` is a long option, which can be passed with `executable_name --trim`
| - `-t` is a short option, which can be passed with `executable_name -t`
| - `trim` is an argument, which can be passed with `executable_name trim`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -66,7 +66,7 @@ case class BooleanArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -74,7 +74,7 @@ case class BooleanArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -84,7 +84,7 @@ case class BooleanArgument(
"""- name: --my_boolean
| type: boolean
| example: true
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
example: OneOrMore[Boolean] = Nil,
@@ -94,7 +94,7 @@ case class BooleanArgument(
"""- name: --my_boolean
| type: boolean
| default: true
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
default: OneOrMore[Boolean] = Nil,
@@ -104,7 +104,7 @@ case class BooleanArgument(
"""- name: --my_boolean
| type: boolean
| required: true
- |""".stripMargin,
+ |""",
"yaml")
@default("False")
required: Boolean = false,
@@ -117,7 +117,7 @@ case class BooleanArgument(
"""- name: --my_boolean
| type: boolean
| multiple: true
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_boolean=true:true:false", "bash", "Here's an example of how to use this:")
@default("False")
@@ -129,7 +129,7 @@ case class BooleanArgument(
| type: boolean
| multiple: true
| multiple_sep: ";"
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_boolean=true,true,false", "bash", "Here's an example of how to use this:")
@default(";")
@@ -169,7 +169,7 @@ case class BooleanArgument(
| type: boolean_true
| description: Ignore console output
| alternatives: ["-s"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("boolean_true")
case class BooleanTrueArgument(
@@ -179,7 +179,7 @@ case class BooleanTrueArgument(
| - `--silent` is a long option, which can be passed with `executable_name --silent`
| - `-s` is a short option, which can be passed with `executable_name -s`
| - `silent` is an argument, which can be passed with `executable_name silent`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -202,7 +202,7 @@ case class BooleanTrueArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -210,7 +210,7 @@ case class BooleanTrueArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -261,7 +261,7 @@ case class BooleanTrueArgument(
| type: boolean_false
| description: Disable logging
| alternatives: ["-nl"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("boolean_false")
case class BooleanFalseArgument(
@@ -271,7 +271,7 @@ case class BooleanFalseArgument(
| - `--no-log` is a long option, which can be passed with `executable_name --no-log`
| - `-n` is a short option, which can be passed with `executable_name -n`
| - `no-log` is an argument, which can be passed with `executable_name no-log`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -294,7 +294,7 @@ case class BooleanFalseArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -302,7 +302,7 @@ case class BooleanFalseArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
diff --git a/src/main/scala/io/viash/config/arguments/DoubleArgument.scala b/src/main/scala/io/viash/config/arguments/DoubleArgument.scala
index 3448b1649..5a5c2f4e8 100644
--- a/src/main/scala/io/viash/config/arguments/DoubleArgument.scala
+++ b/src/main/scala/io/viash/config/arguments/DoubleArgument.scala
@@ -29,7 +29,7 @@ import io.viash.schemas._
| default: 1.5
| description: Litres of fluid to process
| alternatives: ["-l"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("double")
case class DoubleArgument(
@@ -39,7 +39,7 @@ case class DoubleArgument(
| - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value`
| - `-f` is a short option, which can be passed with `executable_name -f value`
| - `foo` is an argument, which can be passed with `executable_name value`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -62,7 +62,7 @@ case class DoubleArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -70,7 +70,7 @@ case class DoubleArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -80,7 +80,7 @@ case class DoubleArgument(
"""- name: --my_double
| type: double
| example: 5.8
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
example: OneOrMore[Double] = Nil,
@@ -90,7 +90,7 @@ case class DoubleArgument(
"""- name: --my_double
| type: double
| default: 5.8
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
default: OneOrMore[Double] = Nil,
@@ -100,7 +100,7 @@ case class DoubleArgument(
"""- name: --my_double
| type: double
| required: true
- |""".stripMargin,
+ |""",
"yaml")
@default("False")
required: Boolean = false,
@@ -110,7 +110,7 @@ case class DoubleArgument(
"""- name: --my_double
| type: double
| min: 25.5
- |""".stripMargin,
+ |""",
"yaml")
min: Option[Double] = None,
@@ -119,8 +119,8 @@ case class DoubleArgument(
"""- name: --my_double
| type: double
| max: 80.4
- |""".stripMargin,
- "yaml")
+ |""",
+ "yaml")
max: Option[Double] = None,
@undocumented
@@ -131,7 +131,7 @@ case class DoubleArgument(
"""- name: --my_double
| type: double
| multiple: true
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_double=5.8:22.6:200.4", "bash", "Here's an example of how to use this:")
@default("False")
@@ -143,7 +143,7 @@ case class DoubleArgument(
| type: double
| multiple: true
| multiple_sep: ";"
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_double=5.8,22.6,200.4", "bash", "Here's an example of how to use this:")
@default(";")
diff --git a/src/main/scala/io/viash/config/arguments/FileArgument.scala b/src/main/scala/io/viash/config/arguments/FileArgument.scala
index 02660fcef..052ab803b 100644
--- a/src/main/scala/io/viash/config/arguments/FileArgument.scala
+++ b/src/main/scala/io/viash/config/arguments/FileArgument.scala
@@ -30,7 +30,7 @@ import io.viash.schemas._
| must_exist: true
| description: CSV file to read contents from
| alternatives: ["-i"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("file")
case class FileArgument(
@@ -40,7 +40,7 @@ case class FileArgument(
| - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value`
| - `-f` is a short option, which can be passed with `executable_name -f value`
| - `foo` is an argument, which can be passed with `executable_name value`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -63,7 +63,7 @@ case class FileArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -71,7 +71,7 @@ case class FileArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -81,7 +81,7 @@ case class FileArgument(
"""- name: --my_file
| type: file
| example: data.csv
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
example: OneOrMore[Path] = Nil,
@@ -91,18 +91,19 @@ case class FileArgument(
"""- name: --my_file
| type: file
| default: data.csv
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
default: OneOrMore[Path] = Nil,
- @description("Checks whether the file or folder exists. For input files, this check will happen " +
- "before the execution of the script, while for output files the check will happen afterwards.")
+ @description(
+ """Checks whether the file or folder exists. For input files, this check will happen
+ |before the execution of the script, while for output files the check will happen afterwards.""")
@example(
"""- name: --my_file
| type: file
| must_exist: true
- |""".stripMargin,
+ |""",
"yaml")
@default("True")
must_exist: Boolean = true,
@@ -113,7 +114,7 @@ case class FileArgument(
| type: file
| direction: output
| create_parent: true
- |""".stripMargin,
+ |""",
"yaml")
@default("True")
create_parent: Boolean = true,
@@ -123,7 +124,7 @@ case class FileArgument(
"""- name: --my_file
| type: file
| required: true
- |""".stripMargin,
+ |""",
"yaml")
@default("False")
required: Boolean = false,
@@ -133,7 +134,7 @@ case class FileArgument(
"""- name: --my_output_file
| type: file
| direction: output
- |""".stripMargin,
+ |""",
"yaml")
@default("Input")
direction: Direction = Input,
@@ -152,12 +153,12 @@ case class FileArgument(
| automatically attempt to expand the expression.
|
|Other output arguments (e.g. integer, double, ...) are not supported yet.
- |""".stripMargin)
+ |""")
@example(
"""- name: --my_files
| type: file
| multiple: true
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_files=firstFile.csv:anotherFile.csv:yetAnother.csv", "bash", "Here's an example of how to use this:")
@default("False")
@@ -169,7 +170,7 @@ case class FileArgument(
| type: file
| multiple: true
| multiple_sep: ";"
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_files=firstFile.csv,anotherFile.csv,yetAnother.csv", "bash", "Here's an example of how to use this:")
@default(";")
diff --git a/src/main/scala/io/viash/config/arguments/IntegerArgument.scala b/src/main/scala/io/viash/config/arguments/IntegerArgument.scala
index 3895c8925..7e9f22cda 100644
--- a/src/main/scala/io/viash/config/arguments/IntegerArgument.scala
+++ b/src/main/scala/io/viash/config/arguments/IntegerArgument.scala
@@ -29,7 +29,7 @@ import io.viash.schemas._
| default: 16
| description: Amount of CPU cores to use
| alternatives: ["-c"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("integer")
case class IntegerArgument(
@@ -39,7 +39,7 @@ case class IntegerArgument(
| - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value`
| - `-f` is a short option, which can be passed with `executable_name -f value`
| - `foo` is an argument, which can be passed with `executable_name value`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -62,7 +62,7 @@ case class IntegerArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -70,7 +70,7 @@ case class IntegerArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -80,7 +80,7 @@ case class IntegerArgument(
"""- name: --my_integer
| type: integer
| example: 100
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
example: OneOrMore[Int] = Nil,
@@ -90,7 +90,7 @@ case class IntegerArgument(
"""- name: --my_integer
| type: integer
| default: 100
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
default: OneOrMore[Int] = Nil,
@@ -100,7 +100,7 @@ case class IntegerArgument(
"""- name: --my_integer
| type: integer
| required: true
- |""".stripMargin,
+ |""",
"yaml")
@default("False")
required: Boolean = false,
@@ -110,7 +110,7 @@ case class IntegerArgument(
"""- name: --values
| type: integer
| choices: [1024, 2048, 4096]
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
choices: List[Int] = Nil,
@@ -120,7 +120,7 @@ case class IntegerArgument(
"""- name: --my_integer
| type: integer
| min: 50
- |""".stripMargin,
+ |""",
"yaml")
min: Option[Int] = None,
@@ -129,7 +129,7 @@ case class IntegerArgument(
"""- name: --my_integer
| type: integer
| max: 150
- |""".stripMargin,
+ |""",
"yaml")
max: Option[Int] = None,
@@ -141,7 +141,7 @@ case class IntegerArgument(
"""- name: --my_integer
| type: integer
| multiple: true
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:")
@default("False")
@@ -153,7 +153,7 @@ case class IntegerArgument(
| type: integer
| multiple: true
| multiple_sep: ";"
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:")
@default(";")
diff --git a/src/main/scala/io/viash/config/arguments/LongArgument.scala b/src/main/scala/io/viash/config/arguments/LongArgument.scala
index 075b9e1f0..502b64a54 100644
--- a/src/main/scala/io/viash/config/arguments/LongArgument.scala
+++ b/src/main/scala/io/viash/config/arguments/LongArgument.scala
@@ -29,7 +29,7 @@ import io.viash.schemas._
| default: 16
| description: Amount of CPU cores to use
| alternatives: ["-c"]
- |""".stripMargin,
+ |""",
"yaml")
@since("Viash 0.6.1")
@subclass("long")
@@ -40,7 +40,7 @@ case class LongArgument(
| - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value`
| - `-f` is a short option, which can be passed with `executable_name -f value`
| - `foo` is an argument, which can be passed with `executable_name value`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -63,7 +63,7 @@ case class LongArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -71,7 +71,7 @@ case class LongArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -81,7 +81,7 @@ case class LongArgument(
"""- name: --my_long
| type: long
| example: 100
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
example: OneOrMore[Long] = Nil,
@@ -91,7 +91,7 @@ case class LongArgument(
"""- name: --my_long
| type: long
| default: 100
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
default: OneOrMore[Long] = Nil,
@@ -101,7 +101,7 @@ case class LongArgument(
"""- name: --my_long
| type: long
| required: true
- |""".stripMargin,
+ |""",
"yaml")
@default("False")
required: Boolean = false,
@@ -111,7 +111,7 @@ case class LongArgument(
"""- name: --values
| type: long
| choices: [1024, 2048, 4096]
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
choices: List[Long] = Nil,
@@ -121,7 +121,7 @@ case class LongArgument(
"""- name: --my_long
| type: long
| min: 50
- |""".stripMargin,
+ |""",
"yaml")
min: Option[Long] = None,
@@ -130,7 +130,7 @@ case class LongArgument(
"""- name: --my_long
| type: long
| max: 150
- |""".stripMargin,
+ |""",
"yaml")
max: Option[Long] = None,
@@ -142,7 +142,7 @@ case class LongArgument(
"""- name: --my_long
| type: long
| multiple: true
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:")
@default("False")
@@ -154,7 +154,7 @@ case class LongArgument(
| type: long
| multiple: true
| multiple_sep: ";"
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:")
@default(";")
diff --git a/src/main/scala/io/viash/config/arguments/StringArgument.scala b/src/main/scala/io/viash/config/arguments/StringArgument.scala
index 1a94f7a54..530bbeca2 100644
--- a/src/main/scala/io/viash/config/arguments/StringArgument.scala
+++ b/src/main/scala/io/viash/config/arguments/StringArgument.scala
@@ -29,7 +29,7 @@ import io.viash.schemas._
| default: "meaning of life"
| description: The term to search for
| alternatives: ["-q"]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("string")
case class StringArgument(
@@ -39,7 +39,7 @@ case class StringArgument(
| - `--foo` is a long option, which can be passed with `executable_name --foo=value` or `executable_name --foo value`
| - `-f` is a short option, which can be passed with `executable_name -f value`
| - `foo` is an argument, which can be passed with `executable_name value`
- |""".stripMargin)
+ |""")
name: String,
@description("List of alternative format variations for this argument.")
@@ -62,7 +62,7 @@ case class StringArgument(
@example(
"""description: |
| A (multiline) description of the purpose of
- | this argument.""".stripMargin, "yaml")
+ | this argument.""", "yaml")
@default("Empty")
description: Option[String] = None,
@@ -70,7 +70,7 @@ case class StringArgument(
@example(
"""info:
| category: cat1
- | labels: [one, two, three]""".stripMargin, "yaml")
+ | labels: [one, two, three]""", "yaml")
@since("Viash 0.6.3")
@default("Empty")
info: Json = Json.Null,
@@ -80,7 +80,7 @@ case class StringArgument(
"""- name: --my_string
| type: string
| example: "Hello World"
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
example: OneOrMore[String] = Nil,
@@ -90,7 +90,7 @@ case class StringArgument(
"""- name: --my_string
| type: string
| default: "The answer is 42"
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
default: OneOrMore[String] = Nil,
@@ -100,7 +100,7 @@ case class StringArgument(
"""- name: --my_string
| type: string
| required: true
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
required: Boolean = false,
@@ -110,7 +110,7 @@ case class StringArgument(
"""- name: --language
| type: string
| choices: ["python", "r", "javascript"]
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
choices: List[String] = Nil,
@@ -123,7 +123,7 @@ case class StringArgument(
"""- name: --my_string
| type: string
| multiple: true
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_string=Marc:Susan:Paul", "bash", "Here's an example of how to use this:")
@default("False")
@@ -135,7 +135,7 @@ case class StringArgument(
| type: string
| multiple: true
| multiple_sep: ";"
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription("my_component --my_string=Marc,Susan,Paul", "bash", "Here's an example of how to use this:")
@default(";")
diff --git a/src/main/scala/io/viash/config/arguments/package.scala b/src/main/scala/io/viash/config/arguments/package.scala
index d16d932e0..77a2374ff 100644
--- a/src/main/scala/io/viash/config/arguments/package.scala
+++ b/src/main/scala/io/viash/config/arguments/package.scala
@@ -18,11 +18,8 @@
package io.viash.config
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import cats.syntax.functor._ // for .widen
-import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
-import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
-import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
+import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder
import io.viash.exceptions.ConfigParserSubTypeException
package object arguments {
diff --git a/src/main/scala/io/viash/config/dependencies/Dependency.scala b/src/main/scala/io/viash/config/dependencies/Dependency.scala
index e5b03e338..37401da42 100644
--- a/src/main/scala/io/viash/config/dependencies/Dependency.scala
+++ b/src/main/scala/io/viash/config/dependencies/Dependency.scala
@@ -23,11 +23,12 @@ import io.viash.schemas._
import java.nio.file.Files
import io.viash.ViashNamespace
import io.viash.exceptions.MissingBuildYamlException
+import io.viash.config.ScopeEnum
@description(
"""Specifies a Viash component (script or executable) that should be made available for the code defined in the component.
|The dependency components are collected and copied to the output folder during the Viash build step.
- |""".stripMargin)
+ |""")
@exampleWithDescription(
"""dependencies:
| - name: qc/multiqc
@@ -35,7 +36,7 @@ import io.viash.exceptions.MissingBuildYamlException
| type: github
| repo: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml",
"Definition of dependency with a fully defined repository"
)
@@ -43,7 +44,7 @@ import io.viash.exceptions.MissingBuildYamlException
"""dependencies:
| - name: qc/multiqc
| repository: "github://openpipelines-bio/modules:0.3.0"
- |""".stripMargin,
+ |""",
"yaml",
"Definition of a dependency with a repository using sugar syntax."
)
@@ -51,14 +52,14 @@ import io.viash.exceptions.MissingBuildYamlException
"""dependencies:
| - name: qc/multiqc
| repository: "openpipelines-bio"
- |""".stripMargin,
+ |""",
"yaml",
"Definition of a dependency with a repository defined as 'openpipelines-bio' under `.repositories`."
)
@exampleWithDescription(
"""dependencies:
| - name: qc/multiqc
- |""".stripMargin,
+ |""",
"yaml",
"Definition of a local dependency. This dependency is present in the current code base and will be built when `viash ns build` is run."
)
@@ -76,7 +77,7 @@ case class Dependency(
|This must either be a full definition of the repository or the name of a repository referenced as it is defined under repositories.
|Additionally, the full definition can be specified as a single string where all parameters such as repository type, url, branch or tag are specified.
|Omitting the value sets the dependency as a local dependency, ie. the dependency is available in the same namespace as the component.
- |""".stripMargin)
+ |""")
@default("Empty")
repository: Either[String, Repository] = Right(LocalRepository()),
@@ -92,6 +93,9 @@ case class Dependency(
@internalFunctionality
@description("Location of the dependency component artifacts are written ready to be used.")
writtenPath: Option[String] = None,
+
+ @internalFunctionality
+ internalDependencyTargetScope: ScopeEnum = ScopeEnum.Public
) {
if (alias.isDefined) {
// check functionality name
@@ -119,7 +123,7 @@ case class Dependency(
if (isLocalDependency) {
// Local dependency so it will only exist once the component is built.
// TODO improve this, for one, the runner id should be dynamic
- Some(ViashNamespace.targetOutputPath("", "executable", None, name))
+ Some(ViashNamespace.targetOutputPath("", "executable", internalDependencyTargetScope, None, name))
} else {
// Previous existing dependency. Use the location of the '.build.yaml' to determine the relative location.
val relativePath = Dependency.getRelativePath(fullPath, Paths.get(workRepository.get.localPath))
diff --git a/src/main/scala/io/viash/config/dependencies/GitRepository.scala b/src/main/scala/io/viash/config/dependencies/GitRepository.scala
index f1d924f82..8a8c45413 100644
--- a/src/main/scala/io/viash/config/dependencies/GitRepository.scala
+++ b/src/main/scala/io/viash/config/dependencies/GitRepository.scala
@@ -28,7 +28,7 @@ import java.nio.file.Paths
"""type: git
|uri: git+https://github.com/openpipelines-bio/openpipeline.git
|tag: 0.8.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -36,7 +36,7 @@ import java.nio.file.Paths
|uri: git+https://gitlab.com/viash-io/viash.git
|tag: 0.7.1
|path: src/test/resources/testns
- |""".stripMargin,
+ |""",
"yaml"
)
@subclass("git")
diff --git a/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala
index e8c1d0530..cbf24c383 100644
--- a/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala
+++ b/src/main/scala/io/viash/config/dependencies/GitRepositoryWithName.scala
@@ -29,7 +29,7 @@ import java.nio.file.Paths
|type: git
|uri: git+https://github.com/openpipelines-bio/openpipeline.git
|tag: 0.8.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -38,7 +38,7 @@ import java.nio.file.Paths
|uri: git+https://gitlab.com/viash-io/viash.git
|tag: 0.7.1
|path: src/test/resources/testns
- |""".stripMargin,
+ |""",
"yaml"
)
@subclass("gitwithname")
diff --git a/src/main/scala/io/viash/config/dependencies/GithubRepository.scala b/src/main/scala/io/viash/config/dependencies/GithubRepository.scala
index 016eaf7e0..30022c510 100644
--- a/src/main/scala/io/viash/config/dependencies/GithubRepository.scala
+++ b/src/main/scala/io/viash/config/dependencies/GithubRepository.scala
@@ -28,7 +28,7 @@ import java.nio.file.Paths
"""type: github
|repo: openpipelines-bio/openpipeline
|tag: 0.8.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -36,7 +36,7 @@ import java.nio.file.Paths
|repo: viash-io/viash
|tag: 0.7.1
|path: src/test/resources/testns
- |""".stripMargin,
+ |""",
"yaml"
)
@subclass("github")
diff --git a/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala
index 80c7f1561..4167dd32d 100644
--- a/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala
+++ b/src/main/scala/io/viash/config/dependencies/GithubRepositoryWithName.scala
@@ -29,7 +29,7 @@ import java.nio.file.Paths
|type: github
|repo: openpipelines-bio/openpipeline
|tag: 0.8.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -38,7 +38,7 @@ import java.nio.file.Paths
|repo: viash-io/viash
|tag: 0.7.1
|path: src/test/resources/testns
- |""".stripMargin,
+ |""",
"yaml"
)
@subclass("githubwithname")
diff --git a/src/main/scala/io/viash/config/dependencies/LocalRepository.scala b/src/main/scala/io/viash/config/dependencies/LocalRepository.scala
index 0475f7355..36f54fa4e 100644
--- a/src/main/scala/io/viash/config/dependencies/LocalRepository.scala
+++ b/src/main/scala/io/viash/config/dependencies/LocalRepository.scala
@@ -24,12 +24,12 @@ import java.nio.file.Paths
"""Defines a locally present and available repository.
|This can be used to define components from the same code base as the current component.
|Alternatively, this can be used to refer to a code repository present on the local hard-drive instead of fetchable remotely, for example during development.
- |""".stripMargin
+ |"""
)
@exampleWithDescription(
"""type: local
|path: /additional_code/src
- |""".stripMargin,
+ |""",
"yaml",
"Refer to a local code repository under `additional_code/src` referenced to the Viash Package Config file."
)
diff --git a/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala
index 9df0035c7..3c7bc1ac0 100644
--- a/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala
+++ b/src/main/scala/io/viash/config/dependencies/LocalRepositoryWithName.scala
@@ -24,13 +24,13 @@ import java.nio.file.Paths
"""Defines a locally present and available repository.
|This can be used to define components from the same code base as the current component.
|Alternatively, this can be used to refer to a code repository present on the local hard-drive instead of fetchable remotely, for example during development.
- |""".stripMargin
+ |"""
)
@exampleWithDescription(
"""name: my_local_code
|type: local
|path: /additional_code/src
- |""".stripMargin,
+ |""",
"yaml",
"Refer to a local code repository under `additional_code/src` referenced to the Viash Package Config file."
)
diff --git a/src/main/scala/io/viash/config/dependencies/Repository.scala b/src/main/scala/io/viash/config/dependencies/Repository.scala
index 5afbe1ce3..2d2c3dfdf 100644
--- a/src/main/scala/io/viash/config/dependencies/Repository.scala
+++ b/src/main/scala/io/viash/config/dependencies/Repository.scala
@@ -28,14 +28,14 @@ import java.nio.file.{Path, Paths, Files}
| - @[git](repo_git)
| - @[github](repo_github)
| - @[vsh](repo_vsh)
- |""".stripMargin)
+ |""")
@exampleWithDescription(
"""repositories:
| - name: openpipelines-bio
| type: github
| repo: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml",
"Definition of a repository in the component config or package config.")
@exampleWithDescription(
@@ -45,7 +45,7 @@ import java.nio.file.{Path, Paths, Files}
| type: github
| repo: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml",
"Definition of dependency with a fully defined repository")
@subclass("LocalRepository")
diff --git a/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala b/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala
index 2e640502a..8272c649b 100644
--- a/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala
+++ b/src/main/scala/io/viash/config/dependencies/ViashhubRepository.scala
@@ -28,14 +28,14 @@ import java.nio.file.Paths
"""type: vsh
|repo: biobox
|tag: 0.1.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
"""type: vsh
|repo: openpipelines-bio/openpipeline
|tag: 0.8.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -43,7 +43,7 @@ import java.nio.file.Paths
|repo: openpipelines-bio/openpipeline
|tag: 0.7.1
|path: src/test/resources/testns
- |""".stripMargin,
+ |""",
"yaml"
)
@subclass("viashhub")
diff --git a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala
index eccfa0bd1..b6deb40e2 100644
--- a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala
+++ b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryTrait.scala
@@ -47,10 +47,10 @@ trait ViashhubRepositoryTrait extends AbstractGitRepository {
def getCacheIdentifier(): Option[String] =
Some(s"viashhub-${fullRepo.replace("/", "-")}${tag.map(_.prepended('-')).getOrElse("")}")
- val uri = s"https://viash-hub.com/$fullRepo.git"
- lazy val uri_ssh = s"git@viash-hub.com:$fullRepo.git"
+ val uri = s"https://packages.viash-hub.com/$fullRepo.git"
+ lazy val uri_ssh = s"git@packages.viash-hub.com:$fullRepo.git"
val fakeCredentials = "nouser:nopass@" // obfuscate the credentials a bit so we don't trigger GitGuardian
- lazy val uri_nouser = s"https://${fakeCredentials}viash-hub.com/$fullRepo.git"
+ lazy val uri_nouser = s"https://${fakeCredentials}packages.viash-hub.com/$fullRepo.git"
val storePath = fullRepo // no need to add 'viash-hub.com' to the store path as 'type' (vsh) will be added
}
diff --git a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala
index 66acddf88..f83f68942 100644
--- a/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala
+++ b/src/main/scala/io/viash/config/dependencies/ViashhubRepositoryWithName.scala
@@ -29,7 +29,7 @@ import java.nio.file.Paths
|type: vsh
|repo: biobox
|tag: 0.1.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -37,7 +37,7 @@ import java.nio.file.Paths
|type: vsh
|repo: openpipelines-bio/openpipeline
|tag: 0.8.0
- |""".stripMargin,
+ |""",
"yaml"
)
@example(
@@ -46,7 +46,7 @@ import java.nio.file.Paths
|repo: openpipelines-bio/openpipeline
|tag: 0.7.1
|path: src/test/resources/testns
- |""".stripMargin,
+ |""",
"yaml"
)
@subclass("viashhubwithname")
diff --git a/src/main/scala/io/viash/config/dependencies/package.scala b/src/main/scala/io/viash/config/dependencies/package.scala
index e444735c3..d260a53c5 100644
--- a/src/main/scala/io/viash/config/dependencies/package.scala
+++ b/src/main/scala/io/viash/config/dependencies/package.scala
@@ -18,15 +18,12 @@
package io.viash.config
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
-import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
import cats.syntax.functor._
import dependencies.GithubRepository
package object dependencies {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
// encoders and decoders for Argument
implicit val encodeDependency: Encoder.AsObject[Dependency] = deriveConfiguredEncoderStrict
@@ -67,7 +64,6 @@ package object dependencies {
objJson deepMerge typeJson
}
-
implicit val decodeDependency: Decoder[Dependency] = deriveConfiguredDecoderFullChecks
implicit val decodeGitRepository: Decoder[GitRepository] = deriveConfiguredDecoderFullChecks
implicit val decodeGithubRepository: Decoder[GithubRepository] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala
index 14557cc39..b40086f6f 100644
--- a/src/main/scala/io/viash/config/package.scala
+++ b/src/main/scala/io/viash/config/package.scala
@@ -18,10 +18,6 @@
package io.viash
import io.circe.{Decoder, Encoder, Json, HCursor, JsonObject}
-import io.circe.generic.extras.Configuration
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
-import io.viash.platforms.decodePlatform
-import io.viash.functionality.decodeFunctionality
import io.viash.exceptions.ConfigParserValidationException
import config.ArgumentGroup
@@ -31,13 +27,23 @@ import config.Links
import config.References
import config.Status._
import config.arguments._
+import io.circe.DecodingFailure
+import io.circe.DecodingFailure.Reason.CustomReason
+import io.circe.derivation.{ConfiguredEnumEncoder, ConfiguredEnumDecoder}
package object config {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
- import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._
- import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
- import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
+ import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation
+ import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck.deriveConfiguredDecoderWithValidationCheck
+
+ import io.viash.config.resources.{decodeResource, encodeResource}
+ import io.viash.config.dependencies.{decodeDependency, encodeDependency}
+ import io.viash.config.dependencies.{decodeRepositoryWithName, encodeRepositoryWithName}
+ import io.viash.runners.{decodeRunner, encodeRunner}
+ import io.viash.engines.{decodeEngine, encodeEngine}
+ import io.viash.packageConfig.{decodePackageConfig, encodePackageConfig}
+ import io.viash.platforms.decodePlatform
+ import io.viash.functionality.decodeFunctionality
// encoders and decoders for Config
implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoderStrict[Config]
@@ -267,10 +273,10 @@ package object config {
"Could not convert json to Config."
)
- implicit val encodeBuildInfo: Encoder[BuildInfo] = deriveConfiguredEncoder
+ implicit val encodeBuildInfo: Encoder.AsObject[BuildInfo] = deriveConfiguredEncoder
implicit val decodeBuildInfo: Decoder[BuildInfo] = deriveConfiguredDecoderFullChecks
- // encoder and decoder for Author
+ // encoder and decoder for Author
implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder
implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks
@@ -283,11 +289,21 @@ package object config {
implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoderFullChecks
// encoder and decoder for Status, make string lowercase before decoding
- implicit val encodeStatus: Encoder[Status] = Encoder.encodeEnumeration(Status)
- implicit val decodeStatus: Decoder[Status] = Decoder.decodeEnumeration(Status).prepare {
+ implicit val encodeStatus: Encoder[Status] = ConfiguredEnumEncoder.derive(_.toLowerCase())
+ implicit val decodeStatus: Decoder[Status] = ConfiguredEnumDecoder.derive[Status](_.toLowerCase()).prepare {
_.withFocus(_.mapString(_.toLowerCase()))
}
+ // encoder and decoder for ScopeEnum, make string lowercase before decoding
+ implicit val encodeScopeEnum: Encoder[ScopeEnum] = ConfiguredEnumEncoder.derive(_.toLowerCase())
+ implicit val decodeScopeEnum: Decoder[ScopeEnum] = ConfiguredEnumDecoder.derive[ScopeEnum](_.toLowerCase()).prepare {
+ _.withFocus(_.mapString(_.toLowerCase()))
+ }
+
+ // encoder and decoder for Scope
+ implicit val encodeScope: Encoder.AsObject[Scope] = deriveConfiguredEncoder
+ implicit val decodeScope: Decoder[Scope] = deriveConfiguredDecoderFullChecks
+
implicit val encodeLinks: Encoder.AsObject[Links] = deriveConfiguredEncoderStrict
implicit val decodeLinks: Decoder[Links] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/config/resources/BashScript.scala b/src/main/scala/io/viash/config/resources/BashScript.scala
index b5f8a088f..fc819a37b 100644
--- a/src/main/scala/io/viash/config/resources/BashScript.scala
+++ b/src/main/scala/io/viash/config/resources/BashScript.scala
@@ -27,7 +27,7 @@ import io.viash.config.Config
@description("""An executable Bash script.
|When defined in resources, only the first entry will be executed when running the built component or when running `viash run`.
- |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin)
+ |When defined in test_resources, all entries will be executed during `viash test`.""")
@subclass("bash_script")
case class BashScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/CSharpScript.scala b/src/main/scala/io/viash/config/resources/CSharpScript.scala
index 17b8bc67f..2fc286060 100644
--- a/src/main/scala/io/viash/config/resources/CSharpScript.scala
+++ b/src/main/scala/io/viash/config/resources/CSharpScript.scala
@@ -27,7 +27,7 @@ import io.viash.config.Config
@description("""An executable C# script.
|When defined in resources, only the first entry will be executed when running the built component or when running `viash run`.
- |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin)
+ |When defined in test_resources, all entries will be executed during `viash test`.""")
@subclass("csharp_script")
case class CSharpScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/Executable.scala b/src/main/scala/io/viash/config/resources/Executable.scala
index 6c9f5b6e4..9a1c7ac23 100644
--- a/src/main/scala/io/viash/config/resources/Executable.scala
+++ b/src/main/scala/io/viash/config/resources/Executable.scala
@@ -42,7 +42,7 @@ case class Executable(
def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods = ScriptInjectionMods()
- override def read: Option[String] = None
+ override def readSome: Option[String] = None
override def write(path: Path, overwrite: Boolean): Unit = {}
diff --git a/src/main/scala/io/viash/config/resources/JavaScriptScript.scala b/src/main/scala/io/viash/config/resources/JavaScriptScript.scala
index d11962ed9..b35fd7bb5 100644
--- a/src/main/scala/io/viash/config/resources/JavaScriptScript.scala
+++ b/src/main/scala/io/viash/config/resources/JavaScriptScript.scala
@@ -27,7 +27,7 @@ import io.viash.config.arguments.{Argument, StringArgument, IntegerArgument, Boo
@description("""An executable JavaScript script.
|When defined in resources, only the first entry will be executed when running the built component or when running `viash run`.
- |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin)
+ |When defined in test_resources, all entries will be executed during `viash test`.""")
@subclass("javascript_script")
case class JavaScriptScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/NextflowScript.scala b/src/main/scala/io/viash/config/resources/NextflowScript.scala
index 819e23030..e41d733ed 100644
--- a/src/main/scala/io/viash/config/resources/NextflowScript.scala
+++ b/src/main/scala/io/viash/config/resources/NextflowScript.scala
@@ -30,7 +30,7 @@ import io.viash.helpers.circe._
import io.viash.ViashNamespace
import io.viash.config.dependencies.Dependency
-@description("""A Nextflow script. Work in progress; added mainly for annotation at the moment.""".stripMargin)
+@description("""A Nextflow script. Work in progress; added mainly for annotation at the moment.""")
@subclass("nextflow_script")
case class NextflowScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/PythonScript.scala b/src/main/scala/io/viash/config/resources/PythonScript.scala
index 40bf1030c..feba17735 100644
--- a/src/main/scala/io/viash/config/resources/PythonScript.scala
+++ b/src/main/scala/io/viash/config/resources/PythonScript.scala
@@ -27,7 +27,7 @@ import io.viash.config.Config
@description("""An executable Python script.
|When defined in resources, only the first entry will be executed when running the built component or when running `viash run`.
- |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin)
+ |When defined in test_resources, all entries will be executed during `viash test`.""")
@subclass("python_script")
case class PythonScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/RScript.scala b/src/main/scala/io/viash/config/resources/RScript.scala
index 1d59dda1c..02eef256b 100644
--- a/src/main/scala/io/viash/config/resources/RScript.scala
+++ b/src/main/scala/io/viash/config/resources/RScript.scala
@@ -27,7 +27,7 @@ import io.viash.config.Config
@description("""An executable R script.
|When defined in resources, only the first entry will be executed when running the built component or when running `viash run`.
- |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin)
+ |When defined in test_resources, all entries will be executed during `viash test`.""")
@subclass("r_script")
case class RScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/Resource.scala b/src/main/scala/io/viash/config/resources/Resource.scala
index 1402c533f..f544abfd5 100644
--- a/src/main/scala/io/viash/config/resources/Resource.scala
+++ b/src/main/scala/io/viash/config/resources/Resource.scala
@@ -24,6 +24,7 @@ import io.viash.exceptions.MissingResourceFileException
import java.nio.file.{Path, Paths}
import java.nio.file.NoSuchFileException
import io.viash.schemas._
+import java.io.FileNotFoundException
@description(
"""Resources are files that support the component. The first resource should be @[a script](scripting_languages) that will be executed when the component is run. Additional resources will be copied to the same directory.
@@ -35,14 +36,14 @@ import io.viash.schemas._
| * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`.
| * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`.
| * is_executable: `true` / `false`, whether the resulting resource file should be made executable.
- |""".stripMargin)
+ |""")
@example(
"""resources:
| - type: r_script
| path: script.R
| - type: file
| path: resource1.txt
- |""".stripMargin,
+ |""",
"yaml")
@subclass("BashScript")
@subclass("CSharpScript")
@@ -147,11 +148,17 @@ trait Resource {
basenameRegex.replaceFirstIn(resourcePath, "")
}
- def read: Option[String] = {
- if (text.isDefined) {
- text
- } else {
- IO.readSome(uri.get)
+ def readSome: Option[String] = {
+ text.orElse(IO.readSome(uri.get))
+ }
+
+ def read: String = {
+ try {
+ text.getOrElse(IO.read(uri.get))
+ } catch {
+ case e: FileNotFoundException =>
+ val configString = parent.map(_.toString)
+ throw MissingResourceFileException.apply(uri.get.toString(), configString, e)
}
}
@@ -164,10 +171,7 @@ trait Resource {
}
} catch {
case e: NoSuchFileException =>
- val configString = parent match {
- case Some(uri) => Some(uri.toString)
- case _ => None
- }
+ val configString = parent.map(_.toString)
throw MissingResourceFileException.apply(path.toString(), configString, e)
}
}
diff --git a/src/main/scala/io/viash/config/resources/ScalaScript.scala b/src/main/scala/io/viash/config/resources/ScalaScript.scala
index 4bd7969f9..8a41e1949 100644
--- a/src/main/scala/io/viash/config/resources/ScalaScript.scala
+++ b/src/main/scala/io/viash/config/resources/ScalaScript.scala
@@ -27,7 +27,7 @@ import io.viash.config.Config
@description("""An executable Scala script.
|When defined in resources, only the first entry will be executed when running the built component or when running `viash run`.
- |When defined in test_resources, all entries will be executed during `viash test`.""".stripMargin)
+ |When defined in test_resources, all entries will be executed during `viash test`.""")
@subclass("scala_script")
case class ScalaScript(
path: Option[String] = None,
diff --git a/src/main/scala/io/viash/config/resources/Script.scala b/src/main/scala/io/viash/config/resources/Script.scala
index 92a9d909a..10be5671f 100644
--- a/src/main/scala/io/viash/config/resources/Script.scala
+++ b/src/main/scala/io/viash/config/resources/Script.scala
@@ -26,61 +26,59 @@ trait Script extends Resource {
def generateInjectionMods(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): ScriptInjectionMods
- def readWithInjection(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): Option[String] = {
- read.map(code => {
- val lines = code.split("\n")
- val startIndex = lines.indexWhere(_.contains("VIASH START"))
- val endIndex = lines.indexWhere(_.contains("VIASH END"))
-
- // compute mods
- val mods = generateInjectionMods(argsMetaAndDeps, config)
-
- val viashLines = Array(
- companion.commentStr + " The following code has been auto-generated by Viash.",
- mods.params
- )
-
- val li =
- if (startIndex >= 0 && endIndex >= 0) {
- val Whitespace = raw"^(\s+).*".r
- val viashLinesWDelimiter =
- lines(startIndex) match {
- case Whitespace(prefix) =>
- viashLines.flatMap(_.split("\n")).map(prefix + _)
- case _ => viashLines
- }
-
- lines.slice(0, startIndex + 1) ++
- viashLinesWDelimiter ++
- lines.slice(endIndex, lines.length)
- } else {
- Array(companion.commentStr + companion.commentStr + " VIASH START") ++
- viashLines ++
- Array(companion.commentStr + companion.commentStr + " VIASH END") ++
- lines
- }
- val li2 =
- { if (mods.header.isEmpty()) Array.empty[String] else Array(mods.header) } ++
- li ++
- { if (mods.footer.isEmpty()) Array.empty[String] else Array(mods.footer) }
-
- li2.mkString("\n")
- })
+ def readWithInjection(argsMetaAndDeps: Map[String, List[Argument[_]]], config: Config): String = {
+ val code = read
+ val lines = code.split("\n")
+ val startIndex = lines.indexWhere(_.contains("VIASH START"))
+ val endIndex = lines.indexWhere(_.contains("VIASH END"))
+
+ // compute mods
+ val mods = generateInjectionMods(argsMetaAndDeps, config)
+
+ val viashLines = Array(
+ companion.commentStr + " The following code has been auto-generated by Viash.",
+ mods.params
+ )
+
+ val li =
+ if (startIndex >= 0 && endIndex >= 0) {
+ val Whitespace = raw"^(\s+).*".r
+ val viashLinesWDelimiter =
+ lines(startIndex) match {
+ case Whitespace(prefix) =>
+ viashLines.flatMap(_.split("\n")).map(prefix + _)
+ case _ => viashLines
+ }
+
+ lines.slice(0, startIndex + 1) ++
+ viashLinesWDelimiter ++
+ lines.slice(endIndex, lines.length)
+ } else {
+ Array(companion.commentStr + companion.commentStr + " VIASH START") ++
+ viashLines ++
+ Array(companion.commentStr + companion.commentStr + " VIASH END") ++
+ lines
+ }
+ val li2 =
+ { if (mods.header.isEmpty()) Array.empty[String] else Array(mods.header) } ++
+ li ++
+ { if (mods.footer.isEmpty()) Array.empty[String] else Array(mods.footer) }
+
+ li2.mkString("\n")
}
def readWithoutInjection = {
- read.map(code => {
- val lines = code.split("\n")
- val startIndex = lines.indexWhere(_.contains("VIASH START"))
- val endIndex = lines.indexWhere(_.contains("VIASH END"))
- val li =
- if (startIndex >= 0 && endIndex >= 0) {
- lines.slice(0, startIndex + 1) ++ lines.slice(endIndex, lines.length)
- } else {
- lines
- }
- li.mkString("\n")
- })
+ val code = read
+ val lines = code.split("\n")
+ val startIndex = lines.indexWhere(_.contains("VIASH START"))
+ val endIndex = lines.indexWhere(_.contains("VIASH END"))
+ val li =
+ if (startIndex >= 0 && endIndex >= 0) {
+ lines.slice(0, startIndex + 1) ++ lines.slice(endIndex, lines.length)
+ } else {
+ lines
+ }
+ li.mkString("\n")
}
def command(script: String): String = (companion.executor :+ s"\"$script\"").mkString(" ")
diff --git a/src/main/scala/io/viash/config/resources/package.scala b/src/main/scala/io/viash/config/resources/package.scala
index 2354dd5e0..9a84004ee 100644
--- a/src/main/scala/io/viash/config/resources/package.scala
+++ b/src/main/scala/io/viash/config/resources/package.scala
@@ -18,24 +18,20 @@
package io.viash.config
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
-import cats.syntax.functor._
+import cats.syntax.functor._ // for .widen
-import java.net.URI // for .widen
import io.circe.ACursor
package object resources {
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
- implicit val encodeURI: Encoder[URI] = Encoder.instance {
- uri => Json.fromString(uri.toString)
- }
- implicit val decodeURI: Decoder[URI] = Decoder.instance {
- cursor => cursor.value.as[String].map(new URI(_))
- }
+ // implicit val encodeURI: Encoder[URI] = Encoder.instance {
+ // uri => Json.fromString(uri.toString)
+ // }
+ // implicit val decodeURI: Decoder[URI] = Decoder.instance {
+ // cursor => cursor.value.as[String].map(new URI(_))
+ // }
// encoders and decoders for Object
implicit val encodeBashScript: Encoder.AsObject[BashScript] = deriveConfiguredEncoderStrict[BashScript]
diff --git a/src/main/scala/io/viash/engines/DockerEngine.scala b/src/main/scala/io/viash/engines/DockerEngine.scala
index 1dd07e5f9..e4b6c181e 100644
--- a/src/main/scala/io/viash/engines/DockerEngine.scala
+++ b/src/main/scala/io/viash/engines/DockerEngine.scala
@@ -24,6 +24,7 @@ import io.viash.config.{Config, BuildInfo, Author}
import io.viash.engines.requirements.{Requirements, DockerRequirements}
import io.viash.helpers.{Escaper, Docker}
import io.viash.wrapper.BashWrapper
+import io.viash.helpers.data_structures.listToOneOrMore
import io.viash.schemas._
import io.viash.helpers.DockerImageInfo
@@ -31,7 +32,7 @@ import io.viash.helpers.DockerImageInfo
@description(
"""Run a Viash component on a Docker backend engine.
|By specifying which dependencies your component needs, users will be able to build a docker container from scratch using the setup flag, or pull it from a docker repository.
- |""".stripMargin)
+ |""")
@example(
"""engines:
| - type: docker
@@ -39,7 +40,7 @@ import io.viash.helpers.DockerImageInfo
| setup:
| - type: apt
| packages: [ curl ]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("docker")
final case class DockerEngine(
@@ -108,7 +109,7 @@ final case class DockerEngine(
| - @[yum](yum_req)
|
|The order in which these dependencies are specified determines the order in which they will be installed.
- |""".stripMargin)
+ |""")
@default("Empty")
setup: List[Requirements] = Nil,
diff --git a/src/main/scala/io/viash/engines/Engine.scala b/src/main/scala/io/viash/engines/Engine.scala
index 6862bca58..7c45ad543 100644
--- a/src/main/scala/io/viash/engines/Engine.scala
+++ b/src/main/scala/io/viash/engines/Engine.scala
@@ -26,13 +26,13 @@ import io.viash.schemas._
|
| * @[Docker](docker_engine)
| * @[Native](native_engine)
- |""".stripMargin)
+ |""")
@example(
"""engines:
| - type: docker
| image: "bash:4.0"
| - type: native
- |""".stripMargin,
+ |""",
"yaml")
@subclass("DockerEngine")
@subclass("NativeEngine")
diff --git a/src/main/scala/io/viash/engines/NativeEngine.scala b/src/main/scala/io/viash/engines/NativeEngine.scala
index 0c13ca4af..4525a1e0b 100644
--- a/src/main/scala/io/viash/engines/NativeEngine.scala
+++ b/src/main/scala/io/viash/engines/NativeEngine.scala
@@ -22,11 +22,11 @@ import io.viash.schemas._
@description(
"""Running a Viash component on a native engine means that the script will be executed in your current environment.
|Any dependencies are assumed to have been installed by the user, so the native engine is meant for developers (who know what they're doing) or for simple bash scripts (which have no extra dependencies).
- |""".stripMargin)
+ |""")
@example(
"""engines:
| - type: native
- |""".stripMargin,
+ |""",
"yaml")
@subclass("native")
final case class NativeEngine(
diff --git a/src/main/scala/io/viash/engines/docker/package.scala b/src/main/scala/io/viash/engines/docker/package.scala
index 7e0bacf38..abaeed31d 100644
--- a/src/main/scala/io/viash/engines/docker/package.scala
+++ b/src/main/scala/io/viash/engines/docker/package.scala
@@ -18,7 +18,6 @@
package io.viash.engines
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
package object docker {
import io.viash.helpers.circe._
diff --git a/src/main/scala/io/viash/engines/package.scala b/src/main/scala/io/viash/engines/package.scala
index e75fdca8c..27e35a11f 100644
--- a/src/main/scala/io/viash/engines/package.scala
+++ b/src/main/scala/io/viash/engines/package.scala
@@ -18,13 +18,12 @@
package io.viash
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import cats.syntax.functor._
// for .widen
package object engines {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
+ import io.viash.engines.requirements.{decodeRequirements, encodeRequirements}
implicit val encodeDockerEngine: Encoder.AsObject[DockerEngine] = deriveConfiguredEncoder
implicit val decodeDockerEngine: Decoder[DockerEngine] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala b/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala
index 1fa1543a8..2be4e031b 100644
--- a/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/ApkRequirements.scala
@@ -25,7 +25,7 @@ import io.viash.schemas._
"""setup:
| - type: apk
| packages: [ sl ]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("apk")
case class ApkRequirements(
diff --git a/src/main/scala/io/viash/engines/requirements/AptRequirements.scala b/src/main/scala/io/viash/engines/requirements/AptRequirements.scala
index 80e81bcdb..830e2bc9e 100644
--- a/src/main/scala/io/viash/engines/requirements/AptRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/AptRequirements.scala
@@ -25,7 +25,7 @@ import io.viash.schemas._
"""setup:
| - type: apt
| packages: [ sl ]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("apt")
case class AptRequirements(
diff --git a/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala b/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala
index 40d7e23a6..652f0d7eb 100644
--- a/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/DockerRequirements.scala
@@ -23,11 +23,11 @@ import io.viash.schemas._
@description("Specify which Docker commands should be run during setup.")
@example(
"""setup:
- # - type: docker
- # build_args: "R_VERSION=hello_world"
- # run: |
- # echo 'Run a custom command'
- # echo 'Foo' > /path/to/file.txt""".stripMargin('#'),
+ | - type: docker
+ | build_args: "R_VERSION=hello_world"
+ | run: |
+ | echo 'Run a custom command'
+ | echo 'Foo' > /path/to/file.txt""",
"yaml")
@subclass("docker")
case class DockerRequirements(
@@ -48,8 +48,8 @@ case class DockerRequirements(
@description("Specifies which `RUN` entries to add to the Dockerfile while building it.")
@example("""run: |
- # echo 'Run a custom command'
- # echo 'Foo' > /path/to/file.txt""".stripMargin('#'), "yaml")
+ | echo 'Run a custom command'
+ | echo 'Foo' > /path/to/file.txt""", "yaml")
@default("Empty")
run: OneOrMore[String] = Nil,
diff --git a/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala b/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala
index 9bf3a97ef..67bcc29fd 100644
--- a/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/JavaScriptRequirements.scala
@@ -28,7 +28,7 @@ import io.viash.schemas._
| git: "https://some.git.repository/org/repo"
| github: "owner/repository"
| url: "https://github.com/org/repo/archive/HEAD.zip"
- |""".stripMargin,
+ |""",
"yaml")
@subclass("javascript")
case class JavaScriptRequirements(
diff --git a/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala b/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala
index f96ed4f93..bd29bef7c 100644
--- a/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/PythonRequirements.scala
@@ -27,7 +27,7 @@ import io.viash.schemas._
| pip: numpy
| github: [ jkbr/httpie, foo/bar ]
| url: "https://github.com/some_org/some_pkg/zipball/master"
- |""".stripMargin,
+ |""",
"yaml")
@subclass("python")
case class PythonRequirements(
@@ -87,8 +87,8 @@ case class PythonRequirements(
@description("Specifies a code block to run as part of the build.")
@example("""script: |
- # print("Running custom code")
- # x = 1 + 1 == 2""".stripMargin('#'), "yaml")
+ | print("Running custom code")
+ | x = 1 + 1 == 2""", "yaml")
@default("Empty")
script: OneOrMore[String] = Nil,
diff --git a/src/main/scala/io/viash/engines/requirements/RRequirements.scala b/src/main/scala/io/viash/engines/requirements/RRequirements.scala
index f306099d2..4c03c4ff7 100644
--- a/src/main/scala/io/viash/engines/requirements/RRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/RRequirements.scala
@@ -27,7 +27,7 @@ import io.viash.schemas._
| cran: anndata
| bioc: [ AnnotationDbi, SingleCellExperiment ]
| github: rcannood/SCORPIUS
- |""".stripMargin,
+ |""",
"yaml")
@subclass("r")
case class RRequirements(
@@ -78,8 +78,8 @@ case class RRequirements(
@description("Specifies a code block to run as part of the build.")
@example("""script: |
- # cat("Running custom code\n")
- # install.packages("anndata")""".stripMargin('#'), "yaml")
+ | cat("Running custom code\n")
+ | install.packages("anndata")""", "yaml")
@default("Empty")
script: OneOrMore[String] = Nil,
@@ -87,15 +87,24 @@ case class RRequirements(
@example("bioc_force_install: false", "yaml")
@default("False")
bioc_force_install: Boolean = false,
+
+ @description("Specifies whether to treat warnings as errors. Default: true.")
+ @example("warnings_as_errors: true", "yaml")
+ @default("True")
+ warnings_as_errors: Boolean = true,
`type`: String = "r"
) extends Requirements {
- assert(script.forall(!_.contains("'")), "R requirement '.script' field contains a single quote ('). This is not allowed.")
-
def installCommands: List[String] = {
+ val prefix = if (warnings_as_errors) "options(warn = 2); " else ""
+
+ def runRCode(code: String): String = {
+ s"""Rscript -e '${prefix}${code.replaceAll("'", "'\"'\"'")}'"""
+ }
+
val installRemotes =
if ((packages ::: cran ::: git ::: github ::: gitlab ::: bitbucket ::: svn ::: url).nonEmpty) {
- List("""Rscript -e 'if (!requireNamespace("remotes", quietly = TRUE)) install.packages("remotes")'""")
+ List(runRCode("""if (!requireNamespace("remotes", quietly = TRUE)) install.packages("remotes")"""))
} else {
Nil
}
@@ -112,17 +121,17 @@ case class RRequirements(
val installBiocManager =
if (bioc.nonEmpty) {
- List("""Rscript -e 'if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager")'""")
+ List(runRCode("""if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager")"""))
} else {
Nil
}
val installBioc =
if (bioc.nonEmpty) {
if (bioc_force_install) {
- List(s"""Rscript -e 'BiocManager::install(c("${bioc.mkString("\", \"")}"))'""")
+ List(runRCode(s"""BiocManager::install(c("${bioc.mkString("\", \"")}"))"""))
} else {
bioc.map { biocPackage =>
- s"""Rscript -e 'if (!requireNamespace("$biocPackage", quietly = TRUE)) BiocManager::install("$biocPackage")'"""
+ runRCode(s"""if (!requireNamespace("$biocPackage", quietly = TRUE)) BiocManager::install("$biocPackage")""")
}
}
} else {
@@ -132,13 +141,13 @@ case class RRequirements(
val installers = remotePairs.flatMap {
case (_, Nil) => None
case (str, list) =>
- Some(s"""Rscript -e 'remotes::install_$str(c("${list.mkString("\", \"")}"), repos = "https://cran.rstudio.com")'""")
+ Some(runRCode(s"""remotes::install_$str(c("${list.mkString("\", \"")}"), repos = "https://cran.rstudio.com")"""))
}
val installScript =
if (script.nonEmpty) {
script.map { line =>
- s"""Rscript -e '$line'"""
+ runRCode(line)
}
} else {
Nil
diff --git a/src/main/scala/io/viash/engines/requirements/Requirements.scala b/src/main/scala/io/viash/engines/requirements/Requirements.scala
index b5c8be0eb..aff7dca1b 100644
--- a/src/main/scala/io/viash/engines/requirements/Requirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/Requirements.scala
@@ -30,7 +30,7 @@ import io.viash.schemas._
| - @[R](r_req)
| - @[Ruby](ruby_req)
| - @[yum](yum_req)
- |""".stripMargin)
+ |""")
@subclass("ApkRequirements")
@subclass("AptRequirements")
@subclass("DockerRequirements")
diff --git a/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala b/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala
index fb70029bf..17340e2bc 100644
--- a/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/RubyRequirements.scala
@@ -25,7 +25,7 @@ import io.viash.schemas._
"""setup:
| - type: ruby
| packages: [ rspec ]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("ruby")
case class RubyRequirements(
diff --git a/src/main/scala/io/viash/engines/requirements/YumRequirements.scala b/src/main/scala/io/viash/engines/requirements/YumRequirements.scala
index bd71c2c35..b0f696801 100644
--- a/src/main/scala/io/viash/engines/requirements/YumRequirements.scala
+++ b/src/main/scala/io/viash/engines/requirements/YumRequirements.scala
@@ -25,7 +25,7 @@ import io.viash.schemas._
"""setup:
| - type: yum
| packages: [ sl ]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("yum")
case class YumRequirements(
diff --git a/src/main/scala/io/viash/engines/requirements/package.scala b/src/main/scala/io/viash/engines/requirements/package.scala
index 458aa7cba..c6ce02296 100644
--- a/src/main/scala/io/viash/engines/requirements/package.scala
+++ b/src/main/scala/io/viash/engines/requirements/package.scala
@@ -18,12 +18,10 @@
package io.viash.engines
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import cats.syntax.functor._ // for .widen
package object requirements {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder
implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks
@@ -84,4 +82,4 @@ package object requirements {
decoder(cursor)
}
-}
\ No newline at end of file
+}
diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala
index 50ab7916a..49043729e 100644
--- a/src/main/scala/io/viash/functionality/Functionality.scala
+++ b/src/main/scala/io/viash/functionality/Functionality.scala
@@ -19,7 +19,6 @@ package io.viash.functionality
import io.circe.Json
-import io.circe.generic.extras._
import io.viash.config.arguments._
import io.viash.config.resources._
import io.viash.config.Status._
@@ -32,7 +31,7 @@ import scala.collection.immutable.ListMap
@description(
"""The functionality-part of the config file describes the behaviour of the script in terms of arguments and resources.
|By specifying a few restrictions (e.g. mandatory arguments) and adding some descriptions, Viash will automatically generate a stylish command-line interface for you.
- |""".stripMargin)
+ |""")
@deprecated("Functionality level is deprecated, all functionality fields are now located on the top level of the config file.", "0.9.0", "0.10.0")
case class Functionality(
@description("Name of the component and the filename of the executable when built with `viash build`.")
@@ -49,20 +48,20 @@ case class Functionality(
@description(
"""A list of @[authors](author). An author must at least have a name, but can also have a list of roles, an e-mail address, and a map of custom properties.
- +
- +Suggested values for roles are:
- +
- +| Role | Abbrev. | Description |
- +|------|---------|-------------|
- +| maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. |
- +| author | aut | for persons who have made substantial contributions to the software. |
- +| contributor | ctb| for persons who have made smaller contributions (such as code patches).
- +| datacontributor | dtc | for persons or organisations that contributed data sets for the software
- +| copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body.
- +| funder | fnd | for persons or organizations that furnished financial support for the development of the software
- +
- +The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive.
- +""".stripMargin('+'))
+ |
+ |Suggested values for roles are:
+ |
+ || Role | Abbrev. | Description |
+ ||------|---------|-------------|
+ || maintainer | mnt | for the maintainer of the code. Ideally, exactly one maintainer is specified. |
+ || author | aut | for persons who have made substantial contributions to the software. |
+ || contributor | ctb| for persons who have made smaller contributions (such as code patches).
+ || datacontributor | dtc | for persons or organisations that contributed data sets for the software
+ || copyrightholder | cph | for all copyright holders. This is a legal concept so should use the legal name of an institution or corporate body.
+ || funder | fnd | for persons or organizations that furnished financial support for the development of the software
+ |
+ |The [full list of roles](https://www.loc.gov/marc/relators/relaterm.html) is extremely comprehensive.
+ |""")
@example(
"""authors:
| - name: Jane Doe
@@ -76,7 +75,7 @@ case class Functionality(
| - name: Tim Farbe
| roles: [author]
| email: tim@far.be
- |""".stripMargin, "yaml")
+ |""", "yaml")
@since("Viash 0.3.1")
@default("Empty")
authors: List[Author] = Nil,
@@ -88,7 +87,7 @@ case class Functionality(
| - `description: Description of foo`, a description of the argument group. Multiline descriptions are supported.
| - `arguments: [arg1, arg2, ...]`, list of the arguments.
|
- |""".stripMargin)
+ |""")
@example(
"""argument_groups:
| - name: "Input"
@@ -108,7 +107,7 @@ case class Functionality(
| - name: "--output_optional"
| type: file
| direction: output
- |""".stripMargin,
+ |""",
"yaml")
@exampleWithDescription(
"""component_name
@@ -126,7 +125,7 @@ case class Functionality(
|
| --optional_output
| type: file
- |""".stripMargin,
+ |""",
"bash",
"This results in the following output when calling the component with the `--help` argument:")
@since("Viash 0.5.14")
@@ -143,14 +142,14 @@ case class Functionality(
| * path: `path/to/file`, the path of the input file. Can be a relative or an absolute path, or a URI. Mutually exclusive with `text`.
| * text: ...multiline text..., the content of the resulting file specified as a string. Mutually exclusive with `path`.
| * is_executable: `true` / `false`, whether the resulting resource file should be made executable.
- |""".stripMargin)
+ |""")
@example(
"""resources:
| - type: r_script
| path: script.R
| - type: file
| path: resource1.txt
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
resources: List[Resource] = Nil,
@@ -158,9 +157,9 @@ case class Functionality(
@description("A description of the component. This will be displayed with `--help`.")
@example(
"""description: |
- + This component performs function Y and Z.
- + It is possible to make this a multiline string.
- +""".stripMargin('+'),
+ | This component performs function Y and Z.
+ | It is possible to make this a multiline string.
+ |""",
"yaml")
description: Option[String] = None,
@@ -176,7 +175,7 @@ case class Functionality(
| - type: r_script
| path: tests/test2.R
| - path: resource1.txt
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
test_resources: List[Resource] = Nil,
@@ -185,7 +184,7 @@ case class Functionality(
@example(
"""info:
| twitter: wizzkid
- | classes: [ one, two, three ]""".stripMargin, "yaml")
+ | classes: [ one, two, three ]""", "yaml")
@since("Viash 0.4.0")
@default("Empty")
info: Json = Json.Null,
@@ -199,12 +198,12 @@ case class Functionality(
"""@[Computational requirements](computational_requirements) related to running the component.
|`cpus` specifies the maximum number of (logical) cpus a component is allowed to use., whereas
|`memory` specifies the maximum amount of memory a component is allowed to allicate. Memory units must be
- |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""".stripMargin)
+ |in B, KB, MB, GB, TB or PB for SI units (1000-base), or KiB, MiB, GiB, TiB or PiB for binary IEC units (1024-base).""")
@example(
"""requirements:
| cpus: 5
| memory: 10GB
- |""".stripMargin,
+ |""",
"yaml")
@since("Viash 0.6.0")
@default("Empty")
@@ -218,21 +217,21 @@ case class Functionality(
| type: github
| uri: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml",
"Full specification of a repository")
@exampleWithDescription(
"""dependencies:
| - name: qc/multiqc
| repository: "github://openpipelines-bio/modules:0.3.0"
- |""".stripMargin,
+ |""",
"yaml",
"Full specification of a repository using sugar syntax")
@exampleWithDescription(
"""dependencies:
| - name: qc/multiqc
| repository: "openpipelines-bio"
- |""".stripMargin,
+ |""",
"yaml",
"Reference to a repository fully specified under 'repositories'")
@default("Empty")
@@ -240,14 +239,14 @@ case class Functionality(
@description(
"""(Pre-)defines @[repositories](repository) that can be used as repository in dependencies.
- |Allows reusing repository definitions in case it is used in multiple dependencies.""".stripMargin)
+ |Allows reusing repository definitions in case it is used in multiple dependencies.""")
@example(
"""repositories:
| - name: openpipelines-bio
| type: github
| uri: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
repositories: List[RepositoryWithName] = Nil,
@@ -281,7 +280,7 @@ case class Functionality(
| journal={Baz},
| year={2024}
| }
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
references: References = References(),
@@ -294,7 +293,7 @@ case class Functionality(
| homepage: "https://viash.io"
| documentation: "https://viash.io/reference/"
| issue_tracker: "https://github.com/viash-io/viash/issues"
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
links: Links = Links(),
@@ -312,7 +311,7 @@ case class Functionality(
| - @[boolean](arg_boolean)
| - @[boolean_true](arg_boolean_true)
| - @[boolean_false](arg_boolean_false)
- |""".stripMargin)
+ |""")
@example(
"""arguments:
| - name: --foo
@@ -327,7 +326,7 @@ case class Functionality(
| multiple_sep: ";"
| - name: --bar
| type: string
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
arguments: List[Argument[_]] = Nil,
diff --git a/src/main/scala/io/viash/functionality/package.scala b/src/main/scala/io/viash/functionality/package.scala
index 9901da81a..61eaf22aa 100644
--- a/src/main/scala/io/viash/functionality/package.scala
+++ b/src/main/scala/io/viash/functionality/package.scala
@@ -17,7 +17,6 @@
package io.viash
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.circe.{Decoder, Encoder, Json}
import io.circe.ACursor
@@ -36,10 +35,10 @@ import config.arguments._
package object functionality extends Logging {
// import implicits
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
- import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._
- import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
- import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
+
+ import io.viash.config.{decodeAuthor, decodeArgumentGroup, decodeStatus, decodeComputationalRequirements, decodeReferences, decodeLinks}
+ import io.viash.config.resources.decodeResource
+ import io.viash.config.dependencies.{decodeDependency, decodeRepositoryWithName}
// encoder and decoder for Functionality
// implicit val encodeFunctionality: Encoder.AsObject[Functionality] = deriveConfiguredEncoderStrict[Functionality]
diff --git a/src/main/scala/io/viash/helpers/BuildStatus.scala b/src/main/scala/io/viash/helpers/BuildStatus.scala
index d4a418cd8..65df27788 100644
--- a/src/main/scala/io/viash/helpers/BuildStatus.scala
+++ b/src/main/scala/io/viash/helpers/BuildStatus.scala
@@ -17,11 +17,6 @@
package io.viash.helpers.status
-// object BuildStatus extends Enumeration {
-// type BuildStatus = Value
-// val ParseError, Disabled, BuildError, TestError, TestMissing, Success = Value
-// }
-
sealed trait Status {
val isError: Boolean
val color: String
diff --git a/src/main/scala/io/viash/helpers/DependencyResolver.scala b/src/main/scala/io/viash/helpers/DependencyResolver.scala
index 65a9d4c8c..8bffb000e 100644
--- a/src/main/scala/io/viash/helpers/DependencyResolver.scala
+++ b/src/main/scala/io/viash/helpers/DependencyResolver.scala
@@ -32,6 +32,7 @@ import io.viash.ViashNamespace
import io.viash.config.resources.NextflowScript
import io.viash.exceptions.MissingDependencyException
import io.viash.helpers.circe.Convert
+import io.viash.config.ScopeEnum
object DependencyResolver extends Logging {
@@ -94,14 +95,19 @@ object DependencyResolver extends Logging {
val config =
if (dep.isLocalDependency) {
- findLocalConfig(repo.localPath.toString(), namespaceConfigs, dep.name, runnerId)
+ val t = findLocalConfig(repo.localPath.toString(), namespaceConfigs, dep.name, runnerId)
+ t.map(t => (t._1, t._2, Some(t._3)))
} else {
- findRemoteConfig(repo.localPath.toString(), dep.name, runnerId)
+ val t = findRemoteConfig(repo.localPath.toString(), dep.name, runnerId)
+ t.map(t => (t._1, t._2, Option.empty[Config]))
}
+ val internalDependencyTargetScope = config.flatMap(_._3).flatMap(_.scope.toOption).map(_.target).getOrElse(ScopeEnum.Public)
+
dep.copy(
foundConfigPath = config.map(_._1),
- configInfo = config.map(_._2).getOrElse(Map.empty)
+ configInfo = config.map(_._2).getOrElse(Map.empty),
+ internalDependencyTargetScope = internalDependencyTargetScope
)
}
)(config3)
@@ -121,7 +127,7 @@ object DependencyResolver extends Logging {
if (dep.isLocalDependency) {
// Dependency solving will be done by building the component and dependencies of that component will be handled there.
// However, we have to fill in writtenPath. This will be needed when this built component is used as a dependency and we have to resolve dependencies of dependencies.
- val writtenPath = ViashNamespace.targetOutputPath(output, runnerId, None, dep.name)
+ val writtenPath = ViashNamespace.targetOutputPath(output, runnerId, dep.internalDependencyTargetScope, None, dep.name)
dep.copy(writtenPath = Some(writtenPath))
} else {
// copy the dependency to the output folder
@@ -144,7 +150,7 @@ object DependencyResolver extends Logging {
}
// Find configs from the local repository. These still need to be built so we have to deduce the information we want.
- def findLocalConfig(targetDir: String, namespaceConfigs: List[Config], name: String, runnerId: Option[String]): Option[(String, Map[String, String])] = {
+ def findLocalConfig(targetDir: String, namespaceConfigs: List[Config], name: String, runnerId: Option[String]): Option[(String, Map[String, String], Config)] = {
val config = namespaceConfigs.filter{ c =>
val fullName = c.namespace.fold("")(n => n + "/") + c.name
@@ -177,7 +183,7 @@ object DependencyResolver extends Logging {
("name" -> c.name),
("namespace" -> c.namespace.getOrElse(""))
)
- (path, map ++ map2)
+ (path, map ++ map2, c)
}
}
diff --git a/src/main/scala/io/viash/helpers/Helper.scala b/src/main/scala/io/viash/helpers/Helper.scala
index 92d46792c..5c443b0b6 100644
--- a/src/main/scala/io/viash/helpers/Helper.scala
+++ b/src/main/scala/io/viash/helpers/Helper.scala
@@ -21,6 +21,7 @@ import io.viash.config.Config
import io.viash.Main
import io.viash.config.ArgumentGroup
import io.viash.config.arguments.{Argument, StringArgument, Output, IntegerArgument, LongArgument, DoubleArgument, FileArgument}
+import io.viash.helpers.data_structures.oneOrMoreToList
object Helper {
private val maxWidth: Int = 80
diff --git a/src/main/scala/io/viash/helpers/Logger.scala b/src/main/scala/io/viash/helpers/Logger.scala
index 81d1c7f1a..8a2e5589c 100644
--- a/src/main/scala/io/viash/helpers/Logger.scala
+++ b/src/main/scala/io/viash/helpers/Logger.scala
@@ -19,38 +19,26 @@ package io.viash.helpers
import scala.io.AnsiColor
-object LoggerLevel extends Enumeration {
- type Level = Value
- val Error = Value(3)
- val Warn = Value(4)
- val Info = Value(6)
- val Debug = Value(7)
- val Trace = Value(8)
+enum LoggerLevel(val level: Int):
+ case Error extends LoggerLevel(3)
+ case Warn extends LoggerLevel(4)
+ case Info extends LoggerLevel(6)
+ case Debug extends LoggerLevel(7)
+ case Trace extends LoggerLevel(8)
- val Success = Value(5) // Should have been 6, same as Info. Luckely we had a spare spot between 4 and 6
+ case Success extends LoggerLevel(5) // Should have been 6, same as Info. Luckely we had a spare spot between 4 and 6
- def fromString(level: String): Value = {
- level match {
- case "error" => Error
- case "warn" => Warn
- case "info" => Info
- case "debug" => Debug
- case "trace" => Trace
- case _ => throw new IllegalArgumentException(level)
- }
- }
+object LoggerLevel {
+ def fromString(level: String): LoggerLevel = LoggerLevel.valueOf(level.toLowerCase.capitalize)
}
-object LoggerOutput extends Enumeration {
- type Output = Value
- val StdOut = Value(1)
- val StdErr = Value(2)
-}
+enum LoggerOutput:
+ case StdOut, StdErr
/** Partial logging facade styled alike SLF4J.
* Used Grizzled slf4j as further inspiration and basis.
*/
-class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boolean) {
+class Logger(val name: String, val level: LoggerLevel, val useColor: Boolean) {
import LoggerOutput._
import LoggerLevel._
@@ -74,9 +62,9 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole
@inline final def traceOut(msg: => Any): Unit = _logOut(Trace, msg)
@inline final def successOut(msg: => Any): Unit = _logOut(Success, msg)
- @inline final def isEnabled(level: Level): Boolean = this.level >= level
+ @inline final def isEnabled(level: LoggerLevel): Boolean = this.level.level >= level.level
- @inline private def _colorString(level: Level): String =
+ @inline private def _colorString(level: LoggerLevel): String =
level match {
case Error => AnsiColor.RED
case Warn => AnsiColor.YELLOW
@@ -87,7 +75,7 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole
case Success => AnsiColor.GREEN
}
- @inline private def _log(level: Level, msg: => Any): Unit = {
+ @inline private def _log(level: LoggerLevel, msg: => Any): Unit = {
if (!isEnabled(level)) return
if (useColor)
@@ -96,7 +84,7 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole
Console.err.println(msg.toString())
}
- @inline private def _logOut(level: Level, msg: => Any): Unit = {
+ @inline private def _logOut(level: LoggerLevel, msg: => Any): Unit = {
if (!isEnabled(level)) return
if (useColor)
@@ -105,7 +93,7 @@ class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boole
Console.out.println(msg.toString())
}
- @inline def log(out: Output, level: Level, color: String, msg: => Any): Unit = {
+ @inline def log(out: LoggerOutput, level: LoggerLevel, color: String, msg: => Any): Unit = {
if (!isEnabled(level)) return
val printer =
@@ -148,7 +136,7 @@ trait Logging {
protected def traceOut(msg: => Any): Unit = logger.traceOut(msg)
protected def successOut(msg: => Any): Unit = logger.successOut(msg)
- protected def log(out: LoggerOutput.Output, level: LoggerLevel.Level, color: String, msg: => Any): Unit = logger.log(out, level, color, msg)
+ protected def log(out: LoggerOutput, level: LoggerLevel, color: String, msg: => Any): Unit = logger.log(out, level, color, msg)
}
object Logger {
@@ -156,15 +144,15 @@ object Logger {
val rootLoggerName = "Viash-root-logger"
- def apply(name: String, level: LoggerLevel.Level, useColor: Boolean): Logger = new Logger(name, level, useColor)
+ def apply(name: String, level: LoggerLevel, useColor: Boolean): Logger = new Logger(name, level, useColor)
def apply(name: String): Logger = new Logger(name, getLoggerLevel(name), useColor)
def apply(cls: Class[_]): Logger = apply(cls.getName)
def apply[C: ClassTag](): Logger = apply(classTag[C].runtimeClass.getName)
def rootLogger = apply(rootLoggerName)
- object UseLevelOverride extends util.DynamicVariable[LoggerLevel.Level](LoggerLevel.Info)
- def getLoggerLevel(name: String): LoggerLevel.Level = {
+ object UseLevelOverride extends util.DynamicVariable[LoggerLevel](LoggerLevel.Info)
+ def getLoggerLevel(name: String): LoggerLevel = {
if (name != rootLoggerName) // prevent constructor loop
rootLogger.debug(s"GetLoggerLevel for $name")
diff --git a/src/main/scala/io/viash/helpers/Mirroring.scala b/src/main/scala/io/viash/helpers/Mirroring.scala
new file mode 100644
index 000000000..002b3a528
--- /dev/null
+++ b/src/main/scala/io/viash/helpers/Mirroring.scala
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2020 Data Intuitive
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+package io.viash.helpers
+
+import scala.quoted.*
+import io.viash.schemas.*
+
+inline def typeOf[T]: String = ${ typeOfImpl[T] }
+inline def deprecatedOf[T]: Vector[(String, String, String)] = ${ deprecatedOfImpl[T] }
+inline def removedOf[T]: Vector[(String, String, String)] = ${ removedOfImpl[T] }
+
+inline def fieldsOf[T]: List[String] = ${ fieldsOfImpl[T] }
+inline def internalFunctionalityFieldsOf[T]: List[String] = ${ internalFunctionalityFieldsOfImpl[T] }
+inline def deprecatedFieldsOf[T]: Vector[(String, String, String, String)] = ${ deprecatedFieldsOfImpl[T] }
+inline def removedFieldsOf[T]: Vector[(String, String, String, String)] = ${ removedFieldsOfImpl[T] }
+inline def annotationsOf[T]: List[(String, List[String])] = ${ annotationsOfImpl[T] }
+inline def memberTypeAnnotationsOf[T]: List[(String, String, List[(String, List[String])])] = ${ memberTypeAnnotationsOfImpl[T] }
+inline def historyOf[T]: List[String] = ${ historyOfImpl[T] }
+
+def typeOfImpl[T: Type](using Quotes): Expr[String] =
+ import quotes.reflect.*
+ val typeRepr = TypeRepr.of[T]
+
+ // Use pattern matching to extract a simplified name
+ def simpleName(tpe: TypeRepr): String = tpe match {
+ case AppliedType(tycon, args) if !(args.length == 1 && args.head.typeSymbol.name == "Any") =>
+ // If it's a type constructor with arguments, show it in a readable form
+ s"${simpleName(tycon)}[${args.map(simpleName).mkString(",")}]"
+ case _ =>
+ // Strip the full package name to get the simple type name
+ tpe.typeSymbol.name
+ }
+ Expr(simpleName(typeRepr))
+
+def deprecatedOfImpl[T](using Type[T], Quotes): Expr[Vector[(String, String, String)]] =
+ import quotes.reflect.*
+ val annot = TypeRepr.of[deprecated].typeSymbol
+ val tuple = TypeRepr
+ .of[T]
+ .typeSymbol
+ .getAnnotation(annot)
+ .map:
+ case annot =>
+ val annotExpr = annot.asExprOf[deprecated]
+ '{ ($annotExpr.message, $annotExpr.since, $annotExpr.plannedRemoval) }
+ val list = tuple match {
+ case Some(t) => Seq(t)
+ case None => Nil
+ }
+ val seq: Expr[Seq[(String, String, String)]] = Expr.ofSeq(list)
+ '{ $seq.toVector }
+
+def removedOfImpl[T](using Type[T], Quotes): Expr[Vector[(String, String, String)]] =
+ import quotes.reflect.*
+ val annot = TypeRepr.of[removed].typeSymbol
+ val tuple = TypeRepr
+ .of[T]
+ .typeSymbol
+ .getAnnotation(annot)
+ .map:
+ case annot =>
+ val annotExpr = annot.asExprOf[removed]
+ '{ ($annotExpr.message, $annotExpr.deprecatedSince, $annotExpr.since) }
+ val list = tuple match {
+ case Some(t) => Seq(t)
+ case None => Nil
+ }
+ val seq: Expr[Seq[(String, String, String)]] = Expr.ofSeq(list)
+ '{ $seq.toVector }
+
+def fieldsOfImpl[T: Type](using Quotes): Expr[List[String]] =
+ import quotes.reflect.*
+ val tpe = TypeRepr.of[T].typeSymbol
+ val fieldSymbols = tpe.caseFields.map(_.name)
+ Expr(fieldSymbols)
+
+def internalFunctionalityFieldsOfImpl[T: Type](using Quotes): Expr[List[String]] =
+ import quotes.reflect.*
+ val annot = TypeRepr.of[internalFunctionality].typeSymbol
+ val fieldSymbols = TypeRepr
+ .of[T]
+ .baseClasses
+ .flatMap(_.declaredFields)
+ .collect{case f if f.hasAnnotation(annot) => f.name }
+ Expr(fieldSymbols)
+
+def deprecatedFieldsOfImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] =
+ import quotes.reflect.*
+ val annot = TypeRepr.of[deprecated].typeSymbol
+ val tuples = TypeRepr
+ .of[T]
+ .baseClasses
+ .flatMap(_.declaredFields)
+ .collect:
+ case f if f.hasAnnotation(annot) =>
+ val fieldNameExpr = Expr(f.name.asInstanceOf[String])
+ val annotExpr = f.getAnnotation(annot).get.asExprOf[deprecated]
+ '{ ($fieldNameExpr, $annotExpr.message, $annotExpr.since, $annotExpr.plannedRemoval) }
+ val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples)
+ '{ $seq.toVector }
+
+def removedFieldsOfImpl[T: Type](using Quotes): Expr[Vector[(String, String, String, String)]] =
+ import quotes.reflect.*
+ val annot = TypeRepr.of[removed].typeSymbol
+ val tuples = TypeRepr
+ .of[T]
+ .baseClasses
+ .flatMap(_.declaredFields)
+ .collect:
+ case f if f.hasAnnotation(annot) =>
+ val fieldNameExpr = Expr(f.name.asInstanceOf[String])
+ val annotExpr = f.getAnnotation(annot).get.asExprOf[removed]
+ '{ ($fieldNameExpr, $annotExpr.message, $annotExpr.deprecatedSince, $annotExpr.since) }
+ val seq: Expr[Seq[(String, String, String, String)]] = Expr.ofSeq(tuples)
+ '{ $seq.toVector }
+
+
+
+def annotationsOfImpl[T: Type](using Quotes): Expr[List[(String, List[String])]] =
+ import quotes.reflect.*
+ val tpe = TypeRepr.of[T].typeSymbol
+
+ // Traverse tree information and extract values or lists of values
+ def annotationToStrings(ann: Term): List[String] =
+ ann match {case Apply(_, args) => args.collect{ case Literal(constant) => constant.value.toString.stripMargin }}
+
+ // We're not adding annotations of base classes here.
+ // The base classes should be documented as well and the annotations will clash with the annotations of the specific class.
+ val annots = tpe.annotations
+ .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash"))
+ .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann)))
+
+ Expr(annots)
+
+def memberTypeAnnotationsOfImpl[T: Type](using Quotes): Expr[List[(String, String, List[(String, List[String])])]] = {
+ import quotes.reflect.*
+
+ // Traverse tree information and extract values or lists of values
+ def annotationToStrings(ann: Term): List[String] =
+ ann match {case Apply(_, args) => args.collect{ case Literal(constant) => constant.value.toString.stripMargin }}
+
+ // Use pattern matching to extract a simplified name
+ def simpleName(tpe: TypeRepr): String = tpe match {
+ case AppliedType(tycon, args) if !(args.length == 1 && args.head.typeSymbol.name == "Any") =>
+ // If it's a type constructor with arguments, show it in a readable form
+ s"${simpleName(tycon)}[${args.map(simpleName).mkString(",")}]"
+ case _ =>
+ // Strip the full package name to get the simple type name
+ tpe.typeSymbol.name
+ }
+
+ val tpe = TypeRepr.of[T]
+ val typeSymbol = tpe.typeSymbol
+ val baseClasses = tpe.baseClasses.filter(_.fullName.startsWith("io.viash"))
+
+ // base classes don't have case fields, so we need to get the member fields from the base classes and filter them
+ // only get the fields that are either case fields or have annotations
+ val caseFieldNames = typeSymbol.caseFields.map(_.name)
+ val annotatedFields = typeSymbol.fieldMembers.filter(_.annotations.nonEmpty).map(_.name)
+ val toDocumentFields = (caseFieldNames ++ annotatedFields).distinct
+
+ val annots =
+ baseClasses
+ .map{ case bc =>
+ bc.fieldMembers
+ .filter(m => toDocumentFields.contains(m.name))
+ .map(m =>
+ val name = m.name
+ val mTpe = simpleName(m.termRef.widen)
+ val annotations = m.annotations
+ .filter(_.tpe.typeSymbol.fullName.startsWith("io.viash"))
+ .map(ann => (ann.tpe.typeSymbol.name, annotationToStrings(ann)))
+ (name, mTpe, annotations)
+ )
+ }
+ // flatten the list of lists by name
+ val annotsFlattened = annots.flatten.groupBy(_._1).map{ case (k, v) => (k, v.head._2, v.flatMap(_._3)) }.toList
+
+ Expr(annotsFlattened)
+}
+
+def historyOfImpl[T: Type](using Quotes): Expr[List[String]] = {
+ import quotes.reflect.*
+ val baseClasses = TypeRepr.of[T].baseClasses.map(_.fullName).filter(_.startsWith("io.viash"))
+
+ Expr(baseClasses)
+}
\ No newline at end of file
diff --git a/src/main/scala/io/viash/helpers/SysEnv.scala b/src/main/scala/io/viash/helpers/SysEnv.scala
index 6e30349c0..a016012a5 100644
--- a/src/main/scala/io/viash/helpers/SysEnv.scala
+++ b/src/main/scala/io/viash/helpers/SysEnv.scala
@@ -19,29 +19,34 @@ package io.viash.helpers
import io.viash.schemas._
+trait SysEnvTrait {
+ def viashHome: String
+ def viashVersion: Option[String]
+}
+
@nameOverride("EnvironmentVariables")
@description("Viash checks several environment variables during operation.")
@documentFully
-trait SysEnvTrait {
+case class SysEnvCC(
@nameOverride("VIASH_HOME")
@description(
"""If `VIASH_HOME` is not defined, the fallback `HOME`/.viash is used.
|
|Location where specific downloaded versions of Viash will be cached and run from.
- |""".stripMargin)
- def viashHome: String
+ |""")
+ viashHome: String,
@nameOverride("VIASH_VERSION")
@description(
"""A specific Viash version can be set to run the commands with. If so required, the specific Viash version will be downloaded.
|This is useful when replicating older results or building Viash components that use outdated code.
- |""".stripMargin)
+ |""")
@example(
"""VIASH_VERSION=0.7.0 viash ns build""",
"sh"
)
- def viashVersion: Option[String]
-}
+ viashVersion: Option[String]
+) extends SysEnvTrait
object SysEnv extends SysEnvTrait {
diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala
index cab16076c..6913f6999 100644
--- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala
+++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala
@@ -17,21 +17,19 @@
package io.viash.helpers.circe
-import shapeless.Lazy
-import scala.reflect.runtime.universe._
-
import io.circe.Decoder
-import io.circe.generic.extras.decoding.ConfiguredDecoder
-import io.circe.generic.extras.semiauto.deriveConfiguredDecoder
+import io.circe.derivation.Configuration
+import scala.deriving.Mirror
+import io.viash.helpers.typeOf
object DeriveConfiguredDecoderFullChecks {
- import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._
- import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._
+ import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation
+ import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck.validator
- def deriveConfiguredDecoderFullChecks[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A]
+ inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A]
.validate(
validator[A],
- s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}."
+ s"Could not convert json to ${typeOf[A]}."
)
- .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] )
+ .prepare( checkDeprecation[A] )
}
diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala
index f2ea76df4..b1b4434a9 100644
--- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala
+++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala
@@ -17,24 +17,23 @@
package io.viash.helpers.circe
-import io.circe.generic.extras.semiauto.deriveConfiguredDecoder
-import io.circe.{ Decoder, CursorOp }
-import io.circe.generic.extras.decoding.ConfiguredDecoder
-
-import scala.reflect.runtime.universe._
-import shapeless.Lazy
-
-import io.viash.schemas.ParameterSchema
-import io.circe.ACursor
+import io.circe.{ ACursor, Decoder, CursorOp }
+import io.circe.derivation.{Configuration, ConfiguredDecoder}
+import scala.deriving.Mirror
import io.viash.helpers.Logging
-import io.viash.schemas.CollectedSchemas
+import io.viash.helpers.*
object DeriveConfiguredDecoderWithDeprecationCheck extends Logging {
- private def memberDeprecationCheck(name: String, history: List[CursorOp], parameters: List[ParameterSchema]): Unit = {
- val schema = parameters.find(p => p.name == name).getOrElse(ParameterSchema("", "", "", None, None, None, None, None, None, None, None, false, false))
-
+ // This method doesn't use any mirroring, so it can be called from multiple inlined validators without needing inlining.
+ private def memberDeprecationCheck(
+ name: String,
+ history: List[CursorOp],
+ deprecated: Option[(String, String, String)],
+ removed: Option[(String, String, String)],
+ hasInternalFunctionality: Boolean
+ ): Unit = {
lazy val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".")
lazy val fullHistoryName =
@@ -44,42 +43,46 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging {
s".$historyString.$name"
}
- schema.deprecated match {
+ deprecated match {
case Some(d) =>
- info(s"Warning: $fullHistoryName is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.")
+ info(s"Warning: $fullHistoryName is deprecated: ${d._1} Deprecated since ${d._2}, planned removal ${d._3}.")
case _ =>
}
- schema.removed match {
+ removed match {
case Some(r) =>
- info(s"Error: $fullHistoryName was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.")
+ info(s"Error: $fullHistoryName was removed: ${r._1} Initially deprecated ${r._2}, removed ${r._3}.")
case _ =>
}
- if (schema.hasInternalFunctionality) {
+ if (hasInternalFunctionality) {
error(s"Error: $fullHistoryName is internal functionality.")
throw new RuntimeException(s"Internal functionality used: $fullHistoryName")
}
}
- private def selfDeprecationCheck(parameters: List[ParameterSchema]): Unit = {
- val schema = parameters.find(p => p.name == "__this__").get
+ private inline def selfDeprecationCheck[A]()(using inline A: Mirror.Of[A]): Unit = {
+ val name = typeOf[A]
+ val deprecated = deprecatedOf[A].headOption
+ val removed = removedOf[A].headOption
- schema.deprecated match {
+ deprecated match {
case Some(d) =>
- info(s"Warning: ${schema.`type`} is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.")
+ info(s"Warning: $name is deprecated: ${d._1} Deprecated since ${d._2}, planned removal ${d._3}.")
case _ =>
}
- schema.removed match {
+ removed match {
case Some(r) =>
- info(s"Error: ${schema.`type`} was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.")
+ info(s"Error: $name was removed: ${r._1} Initially deprecated ${r._2}, removed ${r._3}.")
case _ =>
}
}
- //
- def checkDeprecation[A](cursor: ACursor)(implicit tag: TypeTag[A]) : ACursor = {
- val parameters = CollectedSchemas.getParameters[A]()
+ inline def checkDeprecation[A](cursor: ACursor)(using inline A: Mirror.Of[A]) : ACursor = {
+
+ selfDeprecationCheck()
- selfDeprecationCheck(parameters)
+ val df = deprecatedFieldsOf[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap
+ val rf = removedFieldsOf[A].map(t => t._1 -> (t._2, t._3, t._4)).toMap
+ val iff = internalFunctionalityFieldsOf[A]
// check each defined 'key' value
for (key <- cursor.keys.getOrElse(Nil)) {
@@ -91,13 +94,13 @@ object DeriveConfiguredDecoderWithDeprecationCheck extends Logging {
case _ => false
}
if (!isEmpty) {
- memberDeprecationCheck(key, cursor.history, parameters)
+ memberDeprecationCheck(key, cursor.history, df.get(key), rf.get(key), iff.contains(key))
}
}
cursor // return unchanged json info
}
// Use prepare to get raw json data to inspect used fields in the json but we're not performing any changes here
- def deriveConfiguredDecoderWithDeprecationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A]
+ inline def deriveConfiguredDecoderWithDeprecationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A]
.prepare( checkDeprecation[A] )
}
diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala
index 6f08e3925..a4cf280ca 100644
--- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala
+++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala
@@ -17,62 +17,66 @@
package io.viash.helpers.circe
-import io.circe.generic.extras.semiauto.deriveConfiguredDecoder
-import io.circe.{ Decoder, CursorOp }
-import io.circe.generic.extras.decoding.ConfiguredDecoder
+import io.circe.{ Decoder, CursorOp, HCursor, DecodingFailure }
+import io.circe.derivation.{Configuration, ConfiguredDecoder}
+import scala.deriving.Mirror
-import scala.reflect.runtime.universe._
-import shapeless.Lazy
-
-import io.viash.schemas.ParameterSchema
-import io.circe.ACursor
import io.viash.exceptions.ConfigParserSubTypeException
import io.viash.exceptions.ConfigParserValidationException
-import io.circe.HCursor
+import io.viash.helpers.{typeOf, fieldsOf}
object DeriveConfiguredDecoderWithValidationCheck {
- // Validate the json can correctly converted to the required type by actually converting it.
- // Throw an exception when the conversion fails.
- def validator[A](pred: HCursor)(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Boolean = {
- val d = deriveConfiguredDecoder[A]
- val v = d(pred)
+ // This method doesn't use any mirroring, so it can be called from multiple inlined validators without needing inlining.
+ private def validatorError(pred: HCursor, validFields: List[String], typeOf: String)(error: DecodingFailure): Boolean = {
+ val usedFields = pred.value.asObject.map(_.keys.toSeq)
+ val invalidFields = usedFields.map(_.diff(validFields))
- v.fold(error => {
- val usedFields = pred.value.asObject.map(_.keys.toSeq)
- val validFields = typeOf[A].members.filter(m => !m.isMethod).map(_.name.toString.strip()).toSeq
- val invalidFields = usedFields.map(_.diff(validFields))
+ val fieldsHint = invalidFields match {
+ case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}")
+ case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}")
+ case _ => None
+ }
- val fieldsHint = invalidFields match {
- case Some(a) if a.length > 1 => Some(s"Unexpected fields: ${a.mkString(", ")}")
- case Some(a) if a.length == 1 => Some(s"Unexpected field: ${a.head}")
- case _ => None
- }
+ val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".")
- val historyString = error.history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".")
+ val hint = (fieldsHint, historyString, error.message) match {
+ case (Some(a), h, _) if h != "" => Some(s".$h -> $a")
+ case (Some(a), _, _) => Some(a)
+ case (None, h, m) if h != "" => Some(s".$h -> $m")
+ case _ => None
+ }
- val hint = (fieldsHint, historyString, error.message) match {
- case (Some(a), h, _) if h != "" => Some(s".$h -> $a")
- case (Some(a), _, _) => Some(a)
- case (None, h, m) if h != "" => Some(s".$h -> $m")
- case _ => None
- }
+ throw new ConfigParserValidationException(typeOf, pred.value.toString(), hint)
+ }
+
+ // Validate the json can correctly converted to the required type by actually converting it.
+ // Throw an exception when the conversion fails.
+ inline def validator[A](pred: HCursor)(using inline A: Mirror.Of[A], inline configuration: Configuration): Boolean = {
+ val d = deriveConfiguredDecoder[A]
+ // val v = d(pred)
+ // TODO not entirely sure why this is needed instead of just doing `val v = d(pred)`
+ // goes wrong when decoding empty PackageConfig
+ val v = pred match {
+ case pred if pred.value.isNull => Right(null.asInstanceOf[A])
+ case _ => d(pred)
+ }
- throw new ConfigParserValidationException(typeOf[A].baseClasses.head.fullName, pred.value.toString(), hint)
- false
- }, _ => true)
+ v.fold(
+ validatorError(pred, fieldsOf[A], typeOf[A]),
+ _ => true)
}
// Attempts to convert the json to the desired class. Throw an exception if the conversion fails.
- def deriveConfiguredDecoderWithValidationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A]
+ inline def deriveConfiguredDecoderWithValidationCheck[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredDecoder[A]
.validate(
validator[A],
- s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}."
+ s"Could not convert json to ${typeOf[A]}."
)
// Dummy decoder to generate exceptions when an invalid type is specified
// We need a valid class type to be specified
- def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A]
+ inline def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = deriveConfiguredDecoder[A]
.validate(
pred => {
throw new ConfigParserSubTypeException(tpe, validTypes, pred.value.toString())
diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala
index 78e78f585..fcc595aaa 100644
--- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala
+++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredEncoderStrict.scala
@@ -17,26 +17,16 @@
package io.viash.helpers.circe
-import io.circe.{Encoder, Json, HCursor}
-// import io.circe.generic.extras.Configuration
-import io.circe.generic.extras.semiauto.deriveConfiguredEncoder
-import io.circe.generic.extras.encoding.ConfiguredAsObjectEncoder
-
-import scala.reflect.runtime.universe._
-import shapeless.Lazy
-import io.viash.schemas.ParameterSchema
-import io.viash.schemas.CollectedSchemas
+import io.circe.Encoder
+import io.circe.derivation.{Configuration, ConfiguredEncoder}
+import scala.deriving.Mirror
+import io.viash.helpers.internalFunctionalityFieldsOf
object DeriveConfiguredEncoderStrict {
- final def deriveConfiguredEncoderStrict[T](implicit encode: Lazy[ConfiguredAsObjectEncoder[T]], tag: TypeTag[T]) = deriveConfiguredEncoder[T]
+ inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = deriveConfiguredEncoder[A]
.mapJsonObject{ jsonObject =>
- val parameters = CollectedSchemas.getParameters[T]()
- jsonObject.filterKeys( k =>
- parameters
- .find(_.name == k) // find the correct parameter
- .map(!_.hasInternalFunctionality) // check if it has the 'internalFunctionality' annotation
- .getOrElse(true) // fallback, shouldn't really happen
- )
+ val fieldMap = internalFunctionalityFieldsOf[A]
+ jsonObject.filterKeys(k => !fieldMap.contains(k))
}
}
diff --git a/src/main/scala/io/viash/helpers/circe/package.scala b/src/main/scala/io/viash/helpers/circe/package.scala
index aee12b151..20163aa5c 100644
--- a/src/main/scala/io/viash/helpers/circe/package.scala
+++ b/src/main/scala/io/viash/helpers/circe/package.scala
@@ -18,15 +18,22 @@
package io.viash.helpers
import io.circe._
-import io.circe.generic.extras.Configuration
+import io.circe.derivation.{Configuration, ConfiguredDecoder, ConfiguredEncoder}
import java.net.URI
import data_structures.OneOrMore
import java.nio.file.Paths
+import scala.deriving.Mirror
+
package object circe {
implicit val customConfig: Configuration =
Configuration.default.withDefaults.withStrictDecoding
+ inline def deriveConfiguredDecoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredDecoder.derived[A]
+ inline def deriveConfiguredDecoderFullChecks[A](using inline A: Mirror.Of[A], inline configuration: Configuration): Decoder[A] = DeriveConfiguredDecoderFullChecks.deriveConfiguredDecoderFullChecks
+ inline def deriveConfiguredEncoder[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = ConfiguredEncoder.derived[A]
+ inline def deriveConfiguredEncoderStrict[A](using inline A: Mirror.Of[A], inline configuration: Configuration) = DeriveConfiguredEncoderStrict.deriveConfiguredEncoderStrict
+
// encoder and decoder for Either
implicit def encodeEither[A,B](implicit ea: Encoder[A], eb: Encoder[B]): Encoder[Either[A,B]] = {
_.fold(ea(_), eb(_))
diff --git a/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala b/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala
index 06511084a..b537318c4 100644
--- a/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala
+++ b/src/main/scala/io/viash/lenses/AppliedConfigLenses.scala
@@ -28,13 +28,13 @@ object AppliedConfigLenses {
val appliedEnginesLens = GenLens[AppliedConfig](_.engines)
val appliedRunnerLens = GenLens[AppliedConfig](_.runner)
- val enginesLens = configLens ^|-> ConfigLenses.enginesLens
- val runnersLens = configLens ^|-> ConfigLenses.runnersLens
+ val enginesLens = configLens andThen ConfigLenses.enginesLens
+ val runnersLens = configLens andThen ConfigLenses.runnersLens
- val configNameLens = configLens ^|-> ConfigLenses.nameLens
- val configVersionLens = configLens ^|-> ConfigLenses.versionLens
- val configRequirementsLens = configLens ^|-> ConfigLenses.requirementsLens
- val configDependenciesLens = configLens ^|-> ConfigLenses.dependenciesLens
- val configRepositoriesLens = configLens ^|-> ConfigLenses.repositoriesLens
- val configResourcesLens = configLens ^|-> ConfigLenses.resourcesLens
+ val configNameLens = configLens andThen ConfigLenses.nameLens
+ val configVersionLens = configLens andThen ConfigLenses.versionLens
+ val configRequirementsLens = configLens andThen ConfigLenses.requirementsLens
+ val configDependenciesLens = configLens andThen ConfigLenses.dependenciesLens
+ val configRepositoriesLens = configLens andThen ConfigLenses.repositoriesLens
+ val configResourcesLens = configLens andThen ConfigLenses.resourcesLens
}
diff --git a/src/main/scala/io/viash/lenses/ConfigLenses.scala b/src/main/scala/io/viash/lenses/ConfigLenses.scala
index 02358d306..926446ae9 100644
--- a/src/main/scala/io/viash/lenses/ConfigLenses.scala
+++ b/src/main/scala/io/viash/lenses/ConfigLenses.scala
@@ -38,7 +38,8 @@ object ConfigLenses {
val keywordsLens = GenLens[Config](_.keywords)
val licenseLens = GenLens[Config](_.license)
val linksLens = GenLens[Config](_.links)
+ val scopeLens = GenLens[Config](_.scope)
- val linksRepositoryLens = linksLens ^|-> repositoryLens
- val linksDockerRegistryLens = linksLens ^|-> LinksLenses.dockerRegistryLens
+ val linksRepositoryLens = linksLens andThen repositoryLens
+ val linksDockerRegistryLens = linksLens andThen LinksLenses.dockerRegistryLens
}
diff --git a/src/main/scala/io/viash/packageConfig/PackageConfig.scala b/src/main/scala/io/viash/packageConfig/PackageConfig.scala
index 5de9dc6cc..46e75817b 100644
--- a/src/main/scala/io/viash/packageConfig/PackageConfig.scala
+++ b/src/main/scala/io/viash/packageConfig/PackageConfig.scala
@@ -21,6 +21,7 @@ import java.nio.file.{Files, Path, Paths}
import io.viash.schemas._
import io.viash.helpers.data_structures.OneOrMore
+import io.viash.helpers.data_structures.listToOneOrMore
import io.viash.helpers.IO
import io.viash.helpers.circe._
import io.circe.Json
@@ -41,7 +42,7 @@ import io.viash.config.{Author, Links, References}
|config_mods: |
| .runners[.type == 'nextflow'].directives.tag := '$id'
| .runners[.type == 'nextflow'].config.script := 'includeConfig("configs/custom.config")'
- |""".stripMargin, "yaml"
+ |""", "yaml"
)
@since("Viash 0.6.4")
case class PackageConfig(
@@ -71,7 +72,7 @@ case class PackageConfig(
@example(
"""description: |
| A (multiline) description of the purpose of this package
- | and the components it contains.""".stripMargin, "yaml")
+ | and the components it contains.""", "yaml")
@since("Viash 0.9.0")
description: Option[String] = None,
@@ -79,7 +80,7 @@ case class PackageConfig(
@example(
"""info:
| twitter: wizzkid
- | classes: [ one, two, three ]""".stripMargin, "yaml")
+ | classes: [ one, two, three ]""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
info: Json = Json.Null,
@@ -91,7 +92,7 @@ case class PackageConfig(
| type: github
| uri: openpipelines-bio/modules
| tag: 0.3.0
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
@since("Viash 0.9.0")
@@ -136,7 +137,7 @@ case class PackageConfig(
| - name: Tim Farbe
| roles: [author]
| email: tim@far.be
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
authors: List[Author] = Nil,
@@ -170,7 +171,7 @@ case class PackageConfig(
| journal={Baz},
| year={2024}
| }
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
references: References = References(),
@@ -183,7 +184,7 @@ case class PackageConfig(
| homepage: "https://viash.io"
| documentation: "https://viash.io/reference/"
| issue_tracker: "https://github.com/viash-io/viash/issues"
- |""".stripMargin, "yaml")
+ |""", "yaml")
@default("Empty")
@since("Viash 0.9.0")
links: Links = Links(),
@@ -245,7 +246,12 @@ object PackageConfig {
/* PACKAGE 0: converted from json */
// convert Json into ViashPackage
- val pack0 = Convert.jsonToClass[PackageConfig](json, path.toString())
+ // val pack0 = Convert.jsonToClass[PackageConfig](json2, path.toString())
+ // TODO fix empty json getting parsed as 'false' and then failing to create a PackageConfig from that
+ val pack0 = json match {
+ case json if json == Json.False => PackageConfig()
+ case json => Convert.jsonToClass[PackageConfig](json, path.toString())
+ }
/* PACKAGE 1: make resources absolute */
// make paths absolute
diff --git a/src/main/scala/io/viash/packageConfig/package.scala b/src/main/scala/io/viash/packageConfig/package.scala
index f080e814e..f60d2505f 100644
--- a/src/main/scala/io/viash/packageConfig/package.scala
+++ b/src/main/scala/io/viash/packageConfig/package.scala
@@ -21,8 +21,11 @@ import io.circe.{Decoder, Encoder}
package object packageConfig {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
- import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
+
+ import io.viash.config.{decodeAuthor, encodeAuthor}
+ import io.viash.config.{decodeLinks, encodeLinks}
+ import io.viash.config.{decodeReferences, encodeReferences}
+ import io.viash.config.dependencies.{decodeRepositoryWithName, encodeRepositoryWithName}
implicit val encodePackageConfig: Encoder.AsObject[PackageConfig] = deriveConfiguredEncoderStrict
implicit val decodePackageConfig: Decoder[PackageConfig] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/platforms/DockerPlatform.scala b/src/main/scala/io/viash/platforms/DockerPlatform.scala
index 7db7d5c3f..7bb600e66 100644
--- a/src/main/scala/io/viash/platforms/DockerPlatform.scala
+++ b/src/main/scala/io/viash/platforms/DockerPlatform.scala
@@ -27,7 +27,7 @@ import io.viash.engines.docker.{DockerResolveVolume, Automatic}
@description(
"""Run a Viash component on a Docker backend platform.
|By specifying which dependencies your component needs, users will be able to build a docker container from scratch using the setup flag, or pull it from a docker repository.
- |""".stripMargin)
+ |""")
@example(
"""platforms:
| - type: docker
@@ -35,7 +35,7 @@ import io.viash.engines.docker.{DockerResolveVolume, Automatic}
| setup:
| - type: apt
| packages: [ curl ]
- |""".stripMargin,
+ |""",
"yaml")
@deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0")
@subclass("docker")
@@ -96,7 +96,7 @@ case class DockerPlatform(
"""port:
| - 80
| - 8080
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
port: OneOrMore[String] = Nil,
@@ -107,24 +107,24 @@ case class DockerPlatform(
@description(
"""The Docker setup strategy to use when building a container.
- +
- +| Strategy | Description |
- +|-----|----------|
- +| `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy.
- +| `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled.
- +| `ifneedbebuild` | Build the image if it does not exist locally.
- +| `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled.
- +| `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
- +| `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it does not exist.
- +| `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it does not exist.
- +| `ifneedbepull` | If the image does not exist locally, pull the image.
- +| `ifneedbepullelsebuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image from scratch.
- +| `ifneedbepullelsecachedbuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image with caching enabled.
- +| `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
- +| `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet.
- +| `donothing` / `meh` | Do not build or pull anything.
- +
- +""".stripMargin('+'))
+ |
+ || Strategy | Description |
+ ||-----|----------|
+ || `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy.
+ || `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled.
+ || `ifneedbebuild` | Build the image if it does not exist locally.
+ || `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled.
+ || `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
+ || `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it does not exist.
+ || `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it does not exist.
+ || `ifneedbepull` | If the image does not exist locally, pull the image.
+ || `ifneedbepullelsebuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image from scratch.
+ || `ifneedbepullelsecachedbuild` | Do nothing if the image exists locally. Else, try to pull the image from a registry. Otherwise build the image with caching enabled.
+ || `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
+ || `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet.
+ || `donothing` / `meh` | Do not build or pull anything.
+ |
+ |""")
@example("setup_strategy: alwaysbuild", "yaml")
@default("ifneedbepullelsecachedbuild")
setup_strategy: DockerSetupStrategy = IfNeedBePullElseCachedBuild,
@@ -152,7 +152,7 @@ case class DockerPlatform(
| - @[yum](yum_req)
|
|The order in which these dependencies are specified determines the order in which they will be installed.
- |""".stripMargin)
+ |""")
@default("Empty")
setup: List[Requirements] = Nil,
diff --git a/src/main/scala/io/viash/platforms/NativePlatform.scala b/src/main/scala/io/viash/platforms/NativePlatform.scala
index e01a3256d..fb60d6cd2 100644
--- a/src/main/scala/io/viash/platforms/NativePlatform.scala
+++ b/src/main/scala/io/viash/platforms/NativePlatform.scala
@@ -22,11 +22,11 @@ import io.viash.schemas._
@description(
"""Running a Viash component on a native platform means that the script will be executed in your current environment.
|Any dependencies are assumed to have been installed by the user, so the native platform is meant for developers (who know what they're doing) or for simple bash scripts (which have no extra dependencies).
- |""".stripMargin)
+ |""")
@example(
"""platforms:
| - type: native
- |""".stripMargin,
+ |""",
"yaml")
@deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0")
@subclass("native")
diff --git a/src/main/scala/io/viash/platforms/NextflowPlatform.scala b/src/main/scala/io/viash/platforms/NextflowPlatform.scala
index 5df75c206..bf89c470f 100644
--- a/src/main/scala/io/viash/platforms/NextflowPlatform.scala
+++ b/src/main/scala/io/viash/platforms/NextflowPlatform.scala
@@ -23,14 +23,14 @@ import io.viash.runners.nextflow._
/**
* A Platform class for generating Nextflow (DSL2) modules.
*/
-@description("""Platform for generating Nextflow VDSL3 modules.""".stripMargin)
+@description("""Platform for generating Nextflow VDSL3 modules.""")
// todo: add link to guide
@example(
"""platforms:
| - type: nextflow
| directives:
| label: [lowcpu, midmem]
- |""".stripMargin,
+ |""",
"yaml")
@deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0")
@subclass("nextflow")
@@ -45,13 +45,13 @@ case class NextflowPlatform(
// nxf params
@description(
"""@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts.
- |""".stripMargin)
+ |""")
@example(
"""directives:
| container: rocker/r-ver:4.1
| label: highcpu
| cpus: 4
- | memory: 16 GB""".stripMargin,
+ | memory: 16 GB""",
"yaml")
@default("Empty")
directives: NextflowDirectives = NextflowDirectives(),
@@ -66,17 +66,17 @@ case class NextflowPlatform(
|| `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` |
|| `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. If equal to `"state"`, also a `.state.yaml` file will be published in the publish dir. Will throw an error if `params.publishDir` is not defined. | `false` |
|
- |""".stripMargin)
+ |""")
@example(
"""auto:
- | publish: true""".stripMargin,
+ | publish: true""",
"yaml")
@default(
"""simplifyInput: true
|simplifyOutput: false
|transcript: false
|publish: false
- |""".stripMargin)
+ |""")
auto: NextflowAuto = NextflowAuto(),
@description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.")
diff --git a/src/main/scala/io/viash/platforms/Platform.scala b/src/main/scala/io/viash/platforms/Platform.scala
index 196a96d4f..351e0f0ae 100644
--- a/src/main/scala/io/viash/platforms/Platform.scala
+++ b/src/main/scala/io/viash/platforms/Platform.scala
@@ -26,7 +26,7 @@ import io.viash.engines.requirements.Requirements
| * @[Native](platform_native)
| * @[Docker](platform_docker)
| * @[Nextflow](platform_nextflow)
- |""".stripMargin)
+ |""")
@example(
"""platforms:
| - type: docker
@@ -35,7 +35,7 @@ import io.viash.engines.requirements.Requirements
| - type: nextflow
| directives:
| label: [lowcpu, midmem]
- |""".stripMargin,
+ |""",
"yaml")
@deprecated("Use 'engines' and 'runners' instead.", "0.9.0", "0.10.0")
@subclass("NativePlatform")
diff --git a/src/main/scala/io/viash/platforms/package.scala b/src/main/scala/io/viash/platforms/package.scala
index 14dfc3f78..c83ceeb70 100644
--- a/src/main/scala/io/viash/platforms/package.scala
+++ b/src/main/scala/io/viash/platforms/package.scala
@@ -18,12 +18,17 @@
package io.viash
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import cats.syntax.functor._ // for .widen
package object platforms {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
+
+ import io.viash.runners.nextflow.decodeNextflowDirectives
+ import io.viash.engines.docker.decodeResolveVolume
+ import io.viash.runners.executable.decodeSetupStrategy
+ import io.viash.engines.requirements.decodeRequirements
+ import io.viash.runners.nextflow.decodeNextflowAuto
+ import io.viash.runners.nextflow.decodeNextflowConfig
// implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder
implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/runners/ExecutableRunner.scala b/src/main/scala/io/viash/runners/ExecutableRunner.scala
index 53f3d87ba..b07f82f15 100644
--- a/src/main/scala/io/viash/runners/ExecutableRunner.scala
+++ b/src/main/scala/io/viash/runners/ExecutableRunner.scala
@@ -45,12 +45,12 @@ import io.viash.schemas._
|This runner is also used for the @[native](native_engine) engine.
|
|This runner is also used for the @[docker](docker_engine) engine.
- |""".stripMargin)
+ |""")
@example(
"""runners:
| - type: executable
| port: 8080
- |""".stripMargin,
+ |""",
"yaml")
@subclass("executable")
final case class ExecutableRunner(
@@ -64,7 +64,7 @@ final case class ExecutableRunner(
"""port:
| - 80
| - 8080
- |""".stripMargin,
+ |""",
"yaml")
@default("Empty")
port: OneOrMore[String] = Nil,
@@ -75,24 +75,24 @@ final case class ExecutableRunner(
@description(
"""The Docker setup strategy to use when building a docker engine enrivonment.
- +
- +| Strategy | Description |
- +|-----|----------|
- +| `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy.
- +| `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled.
- +| `ifneedbebuild` | Build the image if it does not exist locally.
- +| `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled.
- +| `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
- +| `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it doesn't exist.
- +| `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it doesn't exist.
- +| `ifneedbepull` | If the image does not exist locally, pull the image.
- +| `ifneedbepullelsebuild` | If the image does not exist locally, pull the image. If the image does exist, build it.
- +| `ifneedbepullelsecachedbuild` | If the image does not exist locally, pull the image. If the image does exist, build it with caching enabled.
- +| `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
- +| `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet.
- +| `donothing` / `meh` | Do not build or pull anything.
- +
- +""".stripMargin('+'))
+ |
+ || Strategy | Description |
+ ||-----|----------|
+ || `alwaysbuild` / `build` / `b` | Always build the image from the dockerfile. This is the default setup strategy.
+ || `alwayscachedbuild` / `cachedbuild` / `cb` | Always build the image from the dockerfile, with caching enabled.
+ || `ifneedbebuild` | Build the image if it does not exist locally.
+ || `ifneedbecachedbuild` | Build the image with caching enabled if it does not exist locally, with caching enabled.
+ || `alwayspull` / `pull` / `p` | Try to pull the container from [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
+ || `alwayspullelsebuild` / `pullelsebuild` | Try to pull the image from a registry and build it if it doesn't exist.
+ || `alwayspullelsecachedbuild` / `pullelsecachedbuild` | Try to pull the image from a registry and build it with caching if it doesn't exist.
+ || `ifneedbepull` | If the image does not exist locally, pull the image.
+ || `ifneedbepullelsebuild` | If the image does not exist locally, pull the image. If the image does exist, build it.
+ || `ifneedbepullelsecachedbuild` | If the image does not exist locally, pull the image. If the image does exist, build it with caching enabled.
+ || `push` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry).
+ || `pushifnotpresent` | Push the container to [Docker Hub](https://hub.docker.com) or the @[specified docker registry](docker_registry) if the @[tag](docker_tag) does not exist yet.
+ || `donothing` / `meh` | Do not build or pull anything.
+ |
+ |""")
@example("setup_strategy: alwaysbuild", "yaml")
@default("ifneedbepullelsecachedbuild")
docker_setup_strategy: DockerSetupStrategy = IfNeedBePullElseCachedBuild,
@@ -165,6 +165,12 @@ final case class ExecutableRunner(
| shift 1
| ;;""".stripMargin
+ val helpStrings =
+ s"""Viash built in Engines:
+ | ---engine=ENGINE_ID
+ | Specify the engine to use. Options are: ${engines.map(_.id).mkString(", ")}.
+ | Default: ${engines.head.id}""".stripMargin
+
val typeSetterStrs = engines.groupBy(_.`type`).map{ case (engineType, engineList) =>
s""" ${oneOfEngines(engineList)} ; then
| VIASH_ENGINE_TYPE='${engineType}'""".stripMargin
@@ -179,6 +185,7 @@ final case class ExecutableRunner(
BashWrapperMods(
preParse = preParse,
+ helpStrings = List(("Engine", helpStrings)),
parsers = parsers,
postParse = postParse
)
@@ -337,6 +344,20 @@ final case class ExecutableRunner(
| shift 1
| ;;""".stripMargin
+ val helpStrings =
+ s"""Viash built in Docker:
+ | ---setup=STRATEGY
+ | Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing.
+ | Default: ifneedbepullelsecachedbuild
+ | ---dockerfile
+ | Print the dockerfile to stdout.
+ | ---docker_run_args=ARG
+ | Provide runtime arguments to Docker. See the documentation on `docker run` for more information.
+ | ---docker_image_id
+ | Print the docker image id to stdout.
+ | ---debug
+ | Enter the docker container for debugging purposes.""".stripMargin
+
val setDockerImageId = engines.map { engine =>
s"""[[ "$$VIASH_ENGINE_ID" == '${engine.id}' ]]; then
| VIASH_DOCKER_IMAGE_ID='${engine.getTargetIdentifier(config).toString()}'""".stripMargin
@@ -382,6 +403,7 @@ final case class ExecutableRunner(
BashWrapperMods(
preParse = preParse,
+ helpStrings = List(("Docker", helpStrings)),
parsers = parsers,
postParse = postParse
)
diff --git a/src/main/scala/io/viash/runners/NextflowRunner.scala b/src/main/scala/io/viash/runners/NextflowRunner.scala
index 90df9416b..4e1f8535f 100644
--- a/src/main/scala/io/viash/runners/NextflowRunner.scala
+++ b/src/main/scala/io/viash/runners/NextflowRunner.scala
@@ -33,13 +33,13 @@ import io.viash.config.resources.{Executable, NextflowScript, PlainFile}
@description(
"""Run a Viash component on a Nextflow backend engine.
- |""".stripMargin)
+ |""")
@example(
"""runners:
| - type: nextflow
| directives:
| label: [lowcpu, midmem]
- |""".stripMargin,
+ |""",
"yaml")
@subclass("nextflow")
final case class NextflowRunner(
@@ -53,13 +53,13 @@ final case class NextflowRunner(
// nxf params
@description(
"""@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts.
- |""".stripMargin)
+ |""")
@example(
"""directives:
| container: rocker/r-ver:4.1
| label: highcpu
| cpus: 4
- | memory: 16 GB""".stripMargin,
+ | memory: 16 GB""",
"yaml")
@default("Empty")
directives: NextflowDirectives = NextflowDirectives(),
@@ -74,17 +74,17 @@ final case class NextflowRunner(
|| `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` |
|| `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. If equal to `"state"`, also a `.state.yaml` file will be published in the publish dir. Will throw an error if `params.publishDir` is not defined. | `false` |
|
- |""".stripMargin)
+ |""")
@example(
"""auto:
- | publish: true""".stripMargin,
+ | publish: true""",
"yaml")
@default(
"""simplifyInput: true
|simplifyOutput: false
|transcript: false
|publish: false
- |""".stripMargin)
+ |""")
auto: NextflowAuto = NextflowAuto(),
@description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.")
@@ -217,7 +217,7 @@ final case class NextflowRunner(
// if mainscript is a nextflow workflow
case scr: NextflowScript =>
s"""// user-provided Nextflow code
- |${scr.readWithoutInjection.get.split("\n").mkString("\n|")}
+ |${scr.readWithoutInjection.split("\n").mkString("\n|")}
|
|// inner workflow hook
|def innerWorkflowFactory(args) {
diff --git a/src/main/scala/io/viash/runners/Runner.scala b/src/main/scala/io/viash/runners/Runner.scala
index dc40ec6f8..5af6563e5 100644
--- a/src/main/scala/io/viash/runners/Runner.scala
+++ b/src/main/scala/io/viash/runners/Runner.scala
@@ -27,12 +27,12 @@ import io.viash.config.Config
|
| * @[Executable](executable_runner)
| * @[Nextflow](nextflow_runner)
- |""".stripMargin)
+ |""")
@example(
"""runners:
| - type: executable
| - type: nextflow
- |""".stripMargin,
+ |""",
"yaml")
@subclass("ExecutableRunner")
@subclass("NextflowRunner")
diff --git a/src/main/scala/io/viash/runners/executable/package.scala b/src/main/scala/io/viash/runners/executable/package.scala
index a5d161b06..5f0f0295e 100644
--- a/src/main/scala/io/viash/runners/executable/package.scala
+++ b/src/main/scala/io/viash/runners/executable/package.scala
@@ -18,7 +18,6 @@
package io.viash.runners
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
package object executable {
import io.viash.helpers.circe._
diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala b/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala
index 813309063..faaf2667f 100644
--- a/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala
+++ b/src/main/scala/io/viash/runners/nextflow/NextflowAuto.scala
@@ -25,7 +25,7 @@ case class NextflowAuto(
"""If `true`, an input tuple only containing only a single File (e.g. `["foo", file("in.h5ad")]`) is automatically transformed to a map (i.e. `["foo", [ input: file("in.h5ad") ] ]`).
|
|Default: `true`.
- |""".stripMargin)
+ |""")
@default("True")
simplifyInput: Boolean = true,
@@ -33,7 +33,7 @@ case class NextflowAuto(
"""If `true`, an output tuple containing a map with a File (e.g. `["foo", [ output: file("out.h5ad") ] ]`) is automatically transformed to a map (i.e. `["foo", file("out.h5ad")]`).
|
|Default: `false`.
- |""".stripMargin)
+ |""")
@default("False")
simplifyOutput: Boolean = false,
@@ -43,7 +43,7 @@ case class NextflowAuto(
|Will throw an error if neither are defined.
|
|Default: `false`.
- |""".stripMargin)
+ |""")
@default("False")
transcript: Boolean = false,
@@ -53,7 +53,7 @@ case class NextflowAuto(
|Will throw an error if `params.publishDir` is not defined.
|
|Default: `false`.
- |""".stripMargin)
+ |""")
@default("False")
publish: Either[Boolean, String] = Left(false)
) {
diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala
index ab23fa1c8..40ea310eb 100644
--- a/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala
+++ b/src/main/scala/io/viash/runners/nextflow/NextflowConfig.scala
@@ -20,6 +20,7 @@ package io.viash.runners.nextflow
import scala.collection.immutable.ListMap
import io.viash.schemas._
import io.viash.helpers.data_structures.OneOrMore
+import io.viash.helpers.data_structures.listToOneOrMore
@description("Allows tweaking how the Nextflow Config file is generated.")
@since("Viash 0.7.4")
@@ -32,7 +33,7 @@ case class NextflowConfig(
|
|Conceptually it is possible for a Viash Config to overwrite the full labels parameter, however likely it is more efficient to add additional labels
|in the Viash Package with a config mod.
- |""".stripMargin)
+ |""")
@exampleWithDescription(
"""labels:
| lowmem: "memory = 4.GB"
@@ -43,7 +44,7 @@ case class NextflowConfig(
| highcpu: "cpus = 20"
| vhighmem: "memory = 100.GB"
| vhighcpu: "cpus = 40"
- |""".stripMargin,
+ |""",
"yaml",
"Replace the default labels with a different set of labels")
@exampleWithDescription(
@@ -54,14 +55,14 @@ case class NextflowConfig(
"""config_mods: |
| .runners[.type == "nextflow"].config.labels.lowmem := "memory = 4.GB"
| .runners[.type == "nextflow"].config.labels.lowcpu := "cpus = 4"
- |""".stripMargin,
+ |""",
"viash_package_file",
"Add 'lowmem' and 'lowcpu' to the default labels by using the Viash Package file"
)
@exampleWithDescription(
"""config_mods: |
| .runners[.type == "nextflow"].config.labels := { lowmem: "memory = 4.GB", lowcpu: "cpus = 4", midmem: "memory = 25.GB", midcpu: "cpus = 10", highmem: "memory = 50.GB", highcpu: "cpus = 20", vhighmem: "memory = 100.GB", vhighcpu: "cpus = 40" }
- |""".stripMargin,
+ |""",
"viash_package_file",
"Replace the default labels with a different set of labels by using the Viash Package file"
)
@@ -89,14 +90,14 @@ case class NextflowConfig(
@description(
"""Includes a single string or list of strings into the nextflow.config file.
|This can be used to add custom profiles or include an additional config file.
- |""".stripMargin)
+ |""")
@example(
"""script:
| - |
| profiles {
| ...
| }
- |""".stripMargin,
+ |""",
"yaml")
@example("""script: includeConfig("config.config")""", "yaml")
@default("Empty")
diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala b/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala
index 10a033784..187b5e667 100644
--- a/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala
+++ b/src/main/scala/io/viash/runners/nextflow/NextflowDirectives.scala
@@ -23,13 +23,13 @@ import io.viash.schemas._
// todo: assert contents?
@description(
"""Directives are optional settings that affect the execution of the process.
- |""".stripMargin)
+ |""")
@example(
"""directives:
| container: rocker/r-ver:4.1
| label: highcpu
| cpus: 4
- | memory: 16 GB""".stripMargin,
+ | memory: 16 GB""",
"yaml")
case class NextflowDirectives(
@description(
@@ -38,7 +38,7 @@ case class NextflowDirectives(
|Viash implements this directive as a map with accepted keywords: `type`, `limit`, `request`, and `runtime`.
|
|See [`accelerator`](https://www.nextflow.io/docs/latest/process.html#accelerator).
- |""".stripMargin)
+ |""")
@example("""[ limit: 4, type: "nvidia-tesla-k80" ]""", "yaml")
@default("Empty")
accelerator: Map[String, String] = Map(),
@@ -47,7 +47,7 @@ case class NextflowDirectives(
"""The `afterScript` directive allows you to execute a custom (Bash) snippet immediately after the main process has run. This may be useful to clean up your staging area.
|
|See [`afterScript`](https://www.nextflow.io/docs/latest/process.html#afterscript).
- |""".stripMargin)
+ |""")
@example("""source /cluster/bin/cleanup""", "yaml")
afterScript: Option[String] = None,
@@ -55,7 +55,7 @@ case class NextflowDirectives(
"""The `beforeScript` directive allows you to execute a custom (Bash) snippet before the main process script is run. This may be useful to initialise the underlying cluster environment or for other custom initialisation.
|
|See [`beforeScript`](https://www.nextflow.io/docs/latest/process.html#beforeScript).
- |""".stripMargin)
+ |""")
@example("""source /cluster/bin/setup""", "yaml")
beforeScript: Option[String] = None,
@@ -69,7 +69,7 @@ case class NextflowDirectives(
|Accepted values are: `true`, `false`, `"deep"`, and `"lenient"`.
|
|See [`cache`](https://www.nextflow.io/docs/latest/process.html#cache).
- |""".stripMargin)
+ |""")
@example("true", "yaml")
@example("false", "yaml")
@example(""""deep"""", "yaml")
@@ -82,7 +82,7 @@ case class NextflowDirectives(
|Nextflow automatically sets up an environment for the given package names listed by in the `conda` directive.
|
|See [`conda`](https://www.nextflow.io/docs/latest/process.html#conda).
- |""".stripMargin)
+ |""")
@example(""""bwa=0.7.15"""", "yaml")
@example(""""bwa=0.7.15 fastqc=0.11.5"""", "yaml")
@example("""["bwa=0.7.15", "fastqc=0.11.5"]""", "yaml")
@@ -97,7 +97,7 @@ case class NextflowDirectives(
|Viash implements allows either a string value or a map. In case a map is used, the allowed keys are: `registry`, `image`, and `tag`. The `image` value must be specified.
|
|See [`container`](https://www.nextflow.io/docs/latest/process.html#container).
- |""".stripMargin)
+ |""")
@example(""""foo/bar:tag"""", "yaml")
@exampleWithDescription("""[ registry: "reg", image: "im", tag: "ta" ]""", "yaml", """This is transformed to `"reg/im:ta"`:""")
@exampleWithDescription("""[ image: "im" ]""", "yaml", """This is transformed to `"im:latest"`:""")
@@ -107,7 +107,7 @@ case class NextflowDirectives(
"""The `containerOptions` directive allows you to specify any container execution option supported by the underlying container engine (ie. Docker, Singularity, etc). This can be useful to provide container settings only for a specific process e.g. mount a custom path.
|
|See [`containerOptions`](https://www.nextflow.io/docs/latest/process.html#containeroptions).
- |""".stripMargin)
+ |""")
@example(""""--foo bar"""", "yaml")
@example("""["--foo bar", "-f b"]""", "yaml")
@default("Empty")
@@ -117,7 +117,7 @@ case class NextflowDirectives(
"""The `cpus` directive allows you to define the number of (logical) CPU required by the process' task.
|
|See [`cpus`](https://www.nextflow.io/docs/latest/process.html#cpus).
- |""".stripMargin)
+ |""")
@example("1", "yaml")
@example("10", "yaml")
cpus: Option[Either[Int, String]] = None,
@@ -126,7 +126,7 @@ case class NextflowDirectives(
"""The `disk` directive allows you to define how much local disk storage the process is allowed to use.
|
|See [`disk`](https://www.nextflow.io/docs/latest/process.html#disk).
- |""".stripMargin)
+ |""")
@example(""""1 GB"""", "yaml")
@example(""""2TB"""", "yaml")
@example(""""3.2KB"""", "yaml")
@@ -137,7 +137,7 @@ case class NextflowDirectives(
"""By default the stdout produced by the commands executed in all processes is ignored. By setting the `echo` directive to true, you can forward the process stdout to the current top running process stdout file, showing it in the shell terminal.
|
|See [`echo`](https://www.nextflow.io/docs/latest/process.html#echo).
- |""".stripMargin)
+ |""")
@example("true", "yaml")
@example("false", "yaml")
echo: Option[Either[Boolean, String]] = None,
@@ -154,7 +154,7 @@ case class NextflowDirectives(
|| `retry` | Re-submit for execution a process returning an error condition. |
|
|See [`errorStrategy`](https://www.nextflow.io/docs/latest/process.html#errorstrategy).
- |""".stripMargin)
+ |""")
@example(""""terminate"""", "yaml")
@example(""""finish"""", "yaml")
errorStrategy: Option[String] = None,
@@ -185,7 +185,7 @@ case class NextflowDirectives(
|| uge | Alias for the sge executor. |
|
|See [`executor`](https://www.nextflow.io/docs/latest/process.html#executor).
- |""".stripMargin)
+ |""")
@example(""""local"""", "yaml")
@example(""""sge"""", "yaml")
executor: Option[String] = None,
@@ -194,7 +194,7 @@ case class NextflowDirectives(
"""The `label` directive allows the annotation of processes with mnemonic identifier of your choice.
|
|See [`label`](https://www.nextflow.io/docs/latest/process.html#label).
- |""".stripMargin)
+ |""")
@example(""""big_mem"""", "yaml")
@example(""""big_cpu"""", "yaml")
@example("""["big_mem", "big_cpu"]""", "yaml")
@@ -205,7 +205,7 @@ case class NextflowDirectives(
""" The `machineType` can be used to specify a predefined Google Compute Platform machine type when running using the Google Life Sciences executor.
|
|See [`machineType`](https://www.nextflow.io/docs/latest/process.html#machinetype).
- |""".stripMargin)
+ |""")
@example(""""n1-highmem-8"""", "yaml")
machineType: Option[String] = None,
@@ -213,7 +213,7 @@ case class NextflowDirectives(
"""The `maxErrors` directive allows you to specify the maximum number of times a process can fail when using the `retry` error strategy. By default this directive is disabled.
|
|See [`maxErrors`](https://www.nextflow.io/docs/latest/process.html#maxerrors).
- |""".stripMargin)
+ |""")
@example("1", "yaml")
@example("3", "yaml")
maxErrors: Option[Either[String, Int]] = None,
@@ -224,7 +224,7 @@ case class NextflowDirectives(
|If you want to execute a process in a sequential manner, set this directive to one.
|
|See [`maxForks`](https://www.nextflow.io/docs/latest/process.html#maxforks).
- |""".stripMargin)
+ |""")
@example("1", "yaml")
@example("3", "yaml")
maxForks: Option[Either[String, Int]] = None,
@@ -233,7 +233,7 @@ case class NextflowDirectives(
"""The `maxRetries` directive allows you to define the maximum number of times a process instance can be re-submitted in case of failure. This value is applied only when using the retry error strategy. By default only one retry is allowed.
|
|See [`maxRetries`](https://www.nextflow.io/docs/latest/process.html#maxretries).
- |""".stripMargin)
+ |""")
@example("1", "yaml")
@example("3", "yaml")
maxRetries: Option[Either[String, Int]] = None,
@@ -242,7 +242,7 @@ case class NextflowDirectives(
"""The `memory` directive allows you to define how much memory the process is allowed to use.
|
|See [`memory`](https://www.nextflow.io/docs/latest/process.html#memory).
- |""".stripMargin)
+ |""")
@example(""""1 GB"""", "yaml")
@example(""""2TB"""", "yaml")
@example(""""3.2KB"""", "yaml")
@@ -257,7 +257,7 @@ case class NextflowDirectives(
|In a process definition you can use the `module` directive to load a specific module version to be used in the process execution environment.
|
|See [`module`](https://www.nextflow.io/docs/latest/process.html#module).
- |""".stripMargin)
+ |""")
@example(""""ncbi-blast/2.2.27"""", "yaml")
@example(""""ncbi-blast/2.2.27:t_coffee/10.0"""", "yaml")
@example("""["ncbi-blast/2.2.27", "t_coffee/10.0"]""", "yaml")
@@ -268,7 +268,7 @@ case class NextflowDirectives(
"""The `penv` directive allows you to define the parallel environment to be used when submitting a parallel task to the SGE resource manager.
|
|See [`penv`](https://www.nextflow.io/docs/latest/process.html#penv).
- |""".stripMargin)
+ |""")
@example(""""smp"""", "yaml")
penv: Option[String] = None,
@@ -276,7 +276,7 @@ case class NextflowDirectives(
"""The `pod` directive allows the definition of pods specific settings, such as environment variables, secrets and config maps when using the Kubernetes executor.
|
|See [`pod`](https://www.nextflow.io/docs/latest/process.html#pod).
- |""".stripMargin)
+ |""")
@example("""[ label: "key", value: "val" ]""", "yaml")
@example("""[ annotation: "key", value: "val" ]""", "yaml")
@example("""[ env: "key", value: "val" ]""", "yaml")
@@ -291,7 +291,7 @@ case class NextflowDirectives(
|The allowed values for `mode` are: `symlink`, `rellink`, `link`, `copy`, `copyNoFollow`, `move`.
|
|See [`publishDir`](https://www.nextflow.io/docs/latest/process.html#publishdir).
- |""".stripMargin)
+ |""")
@example("[]", "yaml")
@example("""[ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ]""", "yaml")
@exampleWithDescription(""""/path/to/dir"""", "yaml", """This is transformed to `[[ path: "/path/to/dir" ]]`:""")
@@ -303,7 +303,7 @@ case class NextflowDirectives(
"""The `queue` directory allows you to set the queue where jobs are scheduled when using a grid based executor in your pipeline.
|
|See [`queue`](https://www.nextflow.io/docs/latest/process.html#queue).
- |""".stripMargin)
+ |""")
@example(""""long"""", "yaml")
@example(""""short,long"""", "yaml")
@example("""["short", "long"]""", "yaml")
@@ -314,7 +314,7 @@ case class NextflowDirectives(
"""The `scratch` directive allows you to execute the process in a temporary folder that is local to the execution node.
|
|See [`scratch`](https://www.nextflow.io/docs/latest/process.html#scratch).
- |""".stripMargin)
+ |""")
@example("true", "yaml")
@example(""""/path/to/scratch"""", "yaml")
@example("""'$MY_PATH_TO_SCRATCH'""", "yaml")
@@ -325,7 +325,7 @@ case class NextflowDirectives(
"""The `storeDir` directive allows you to define a directory that is used as a permanent cache for your process results.
|
|See [`storeDir`](https://www.nextflow.io/docs/latest/process.html#storeDir).
- |""".stripMargin)
+ |""")
@example(""""/path/to/storeDir"""", "yaml")
storeDir: Option[String] = None,
@@ -340,7 +340,7 @@ case class NextflowDirectives(
|| rellink | Input files are staged in the process work directory by creating a symbolic link with a relative path for each of them. |
|
|See [`stageInMode`](https://www.nextflow.io/docs/latest/process.html#stageinmode).
- |""".stripMargin)
+ |""")
@example(""""copy"""", "yaml")
@example(""""link"""", "yaml")
stageInMode: Option[String] = None,
@@ -355,7 +355,7 @@ case class NextflowDirectives(
|| rsync | Output files are copied from the scratch directory to the work directory by using the rsync utility. |
|
|See [`stageOutMode`](https://www.nextflow.io/docs/latest/process.html#stageoutmode).
- |""".stripMargin)
+ |""")
@example(""""copy"""", "yaml")
@example(""""link"""", "yaml")
stageOutMode: Option[String] = None,
@@ -366,7 +366,7 @@ case class NextflowDirectives(
|For ease of use, the default tag is set to `"$id"`, which allows tracking the progression of the channel events through the workflow more easily.
|
|See [`tag`](https://www.nextflow.io/docs/latest/process.html#tag).
- |""".stripMargin)
+ |""")
@example(""""foo"""", "yaml")
@default("""'$id'""")
tag: Option[String] = Some("$id"),
@@ -375,7 +375,7 @@ case class NextflowDirectives(
"""The `time` directive allows you to define how long a process is allowed to run.
|
|See [`time`](https://www.nextflow.io/docs/latest/process.html#time).
- |""".stripMargin)
+ |""")
@example(""""1h"""", "yaml")
@example(""""2days"""", "yaml")
@example(""""1day 6hours 3minutes 30seconds"""", "yaml")
diff --git a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala
index c2f3c096c..3f4cee500 100644
--- a/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala
+++ b/src/main/scala/io/viash/runners/nextflow/NextflowHelper.scala
@@ -33,6 +33,8 @@ import java.nio.file.Paths
import io.viash.ViashNamespace
object NextflowHelper {
+ import io.viash.config.encodeConfig
+
private def readSource(s: String) = {
val path = s"io/viash/runners/nextflow/$s"
Source.fromResource(path).getLines().mkString("\n")
@@ -65,7 +67,7 @@ object NextflowHelper {
includeMeta = true,
filterInputs = true
)
- val code = res.readWithInjection(argsAndMeta, config).get
+ val code = res.readWithInjection(argsAndMeta, config)
val escapedCode = Bash.escapeString(code, allowUnescape = true)
.replace("\\", "\\\\")
.replace("'''", "\\'\\'\\'")
diff --git a/src/main/scala/io/viash/runners/nextflow/package.scala b/src/main/scala/io/viash/runners/nextflow/package.scala
index 6dd0a7ff1..218e1cadb 100644
--- a/src/main/scala/io/viash/runners/nextflow/package.scala
+++ b/src/main/scala/io/viash/runners/nextflow/package.scala
@@ -18,11 +18,9 @@
package io.viash.runners
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
package object nextflow {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder
implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/runners/package.scala b/src/main/scala/io/viash/runners/package.scala
index ce8220eee..158e0ec63 100644
--- a/src/main/scala/io/viash/runners/package.scala
+++ b/src/main/scala/io/viash/runners/package.scala
@@ -18,12 +18,15 @@
package io.viash
import io.circe.{Decoder, Encoder, Json}
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import cats.syntax.functor._ // for .widen
package object runners {
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._
+
+ import io.viash.runners.executable.{decodeSetupStrategy, encodeSetupStrategy}
+ import io.viash.runners.nextflow.{decodeNextflowDirectives, encodeNextflowDirectives}
+ import io.viash.runners.nextflow.{decodeNextflowAuto, encodeNextflowAuto}
+ import io.viash.runners.nextflow.{decodeNextflowConfig, encodeNextflowConfig}
implicit val encodeExecutableRunner: Encoder.AsObject[ExecutableRunner] = deriveConfiguredEncoder
implicit val decodeExecutableRunner: Decoder[ExecutableRunner] = deriveConfiguredDecoderFullChecks
diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala
index 35f98614f..a4cafa90a 100644
--- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala
+++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala
@@ -17,22 +17,18 @@
package io.viash.schemas
-import scala.reflect.runtime.universe._
import io.circe.{Encoder, Printer => JsonPrinter}
import io.circe.syntax.EncoderOps
-import io.circe.generic.extras.semiauto.deriveConfiguredEncoder
import io.viash.functionality._
import io.viash.runners._
import io.viash.engines._
import io.viash.platforms._
import io.circe.Json
-import monocle.function.Cons
import io.viash.config.Config
import io.viash.config.BuildInfo
import io.viash.packageConfig.PackageConfig
import io.viash.helpers._
-import scala.collection.immutable.ListMap
import io.viash.runners.nextflow.{NextflowConfig, NextflowAuto, NextflowDirectives}
import io.viash.engines.requirements._
import io.viash.config.arguments._
@@ -43,217 +39,131 @@ import io.viash.config.Author
import io.viash.config.ComputationalRequirements
import io.viash.config.Links
import io.viash.config.References
-
-final case class CollectedSchemas (
- config: Map[String, List[ParameterSchema]],
- functionality: Map[String, List[ParameterSchema]],
- runners: Map[String, List[ParameterSchema]],
- engines: Map[String, List[ParameterSchema]],
- platforms: Map[String, List[ParameterSchema]],
- requirements: Map[String, List[ParameterSchema]],
- arguments: Map[String, List[ParameterSchema]],
- resources: Map[String, List[ParameterSchema]],
- nextflowParameters: Map[String, List[ParameterSchema]],
-)
-
+import io.viash.config.Scope
object CollectedSchemas {
-
- implicit class RichSymbol(s: Symbol) {
- def shortName = s.fullName.split('.').last
- }
-
- case class MemberInfo (
- symbol: Symbol,
- inConstructor: Boolean,
- className: String,
- inheritanceIndex: Int
- ) {
- def fullName = symbol.fullName
- def shortName = symbol.shortName
- }
-
private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true)
import io.viash.helpers.circe._
- import io.viash.helpers.circe.DeriveConfiguredEncoderStrict._
- private implicit val encodeConfigSchema: Encoder.AsObject[CollectedSchemas] = deriveConfiguredEncoder
private implicit val encodeParameterSchema: Encoder.AsObject[ParameterSchema] = deriveConfiguredEncoderStrict
private implicit val encodeDeprecatedOrRemoved: Encoder.AsObject[DeprecatedOrRemovedSchema] = deriveConfiguredEncoder
private implicit val encodeExample: Encoder.AsObject[ExampleSchema] = deriveConfiguredEncoder
- private def getMembers[T: TypeTag](): (Map[String,List[MemberInfo]], List[Symbol]) = {
-
- val name = typeOf[T].typeSymbol.shortName
+ private inline def getMembers[T](): List[ParameterSchema] = {
+ val tpe = typeOf[T]
+ val history = historyOf[T]
+ val annotations = annotationsOf[T]
+ val thisMembers = ParameterSchema("__this__", tpe, history, annotations)
- // Get all members and filter for constructors, first one should be the best (most complete) one
- // Traits don't have constructors
- // Get all parameters and store their short name
- val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String])
-
- val baseClasses = typeOf[T].baseClasses
- .filter(_.fullName.startsWith("io.viash"))
-
- // If we're only getting a abstract class/trait, not a final implementation, use these definitions (otherwise we're left with nothing).
- val documentFully =
- baseClasses.length == 1 &&
- baseClasses.head.isAbstract &&
- baseClasses.head.annotations.exists(a => a.tree.tpe =:= typeOf[documentFully])
-
- val memberNames = typeOf[T].members
- .filter(!_.isMethod || documentFully)
- .map(_.shortName)
- .toSeq
-
- val allMembers = baseClasses
- .zipWithIndex
- .flatMap{ case (baseClass, index) =>
- baseClass.info.members
- .filter(_.fullName.startsWith("io.viash"))
- .filter(m => memberNames.contains(m.shortName))
- .filter(m => !m.info.getClass.toString.endsWith("NullaryMethodType") || index != 0 || documentFully) // Only regular members if base class, otherwise all members
- .map(y => MemberInfo(y, (constructorMembers.contains(y.shortName)), baseClass.fullName, index))
- }
- .groupBy(k => k.shortName)
-
- (allMembers, baseClasses)
+ val memberAnnotations = memberTypeAnnotationsOf[T].map({ case (memberName, memberType, memberAnns) =>
+ ParameterSchema(memberName, memberType, Nil, memberAnns)
+ })
+ thisMembers +: memberAnnotations
}
- lazy val schemaClasses = List(
- getMembers[Config](),
- getMembers[PackageConfig](),
- getMembers[BuildInfo](),
- getMembers[SysEnvTrait](),
-
- getMembers[Functionality](),
- getMembers[Author](),
- getMembers[ComputationalRequirements](),
- getMembers[ArgumentGroup](),
- getMembers[Links](),
- getMembers[References](),
-
- getMembers[Runner](),
- getMembers[ExecutableRunner](),
- getMembers[NextflowRunner](),
-
- getMembers[Engine](),
- getMembers[NativeEngine](),
- getMembers[DockerEngine](),
-
- getMembers[Platform](),
- getMembers[NativePlatform](),
- getMembers[DockerPlatform](),
- getMembers[NextflowPlatform](),
-
- getMembers[Requirements](),
- getMembers[ApkRequirements](),
- getMembers[AptRequirements](),
- getMembers[DockerRequirements](),
- getMembers[JavaScriptRequirements](),
- getMembers[PythonRequirements](),
- getMembers[RRequirements](),
- getMembers[RubyRequirements](),
- getMembers[YumRequirements](),
-
- getMembers[Argument[_]](),
- getMembers[BooleanArgument](),
- getMembers[BooleanTrueArgument](),
- getMembers[BooleanFalseArgument](),
- getMembers[DoubleArgument](),
- getMembers[FileArgument](),
- getMembers[IntegerArgument](),
- getMembers[LongArgument](),
- getMembers[StringArgument](),
-
- getMembers[Resource](),
- getMembers[BashScript](),
- getMembers[CSharpScript](),
- getMembers[Executable](),
- getMembers[JavaScriptScript](),
- getMembers[NextflowScript](),
- getMembers[PlainFile](),
- getMembers[PythonScript](),
- getMembers[RScript](),
- getMembers[ScalaScript](),
-
- getMembers[NextflowDirectives](),
- getMembers[NextflowAuto](),
- getMembers[NextflowConfig](),
-
- getMembers[Dependency](),
- getMembers[Repository](),
- getMembers[LocalRepository](),
- getMembers[GitRepository](),
- getMembers[GithubRepository](),
- getMembers[ViashhubRepository](),
- getMembers[RepositoryWithName](),
- getMembers[LocalRepositoryWithName](),
- getMembers[GitRepositoryWithName](),
- getMembers[GithubRepositoryWithName](),
- getMembers[ViashhubRepositoryWithName](),
- )
-
- private def trimTypeName(s: String) = {
- // first: io.viash.helpers.data_structures.OneOrMore[String] -> OneOrMore[String]
- // second: List[io.viash.platforms.requirements.Requirements] -> List[Requirements]
- // third: Either[String,io.viash.functionality.dependencies.Repository] -> Either[String,Repository]
- s
- .replaceAll("""^(\w*\.)*""", "")
- .replaceAll("""(\w*)\[[\w\.]*?(\w*)(\[_\])?\]""", "$1[$2]")
- .replaceAll("""(\w*)\[[\w\.]*?(\w*),[\w\.]*?(\w*)\]""", "$1[$2,$3]")
+ // split the data in two parts to avoid the compiler complaining about the size
+ object memberData_part1 {
+ val part = List(
+ getMembers[Config](),
+ getMembers[PackageConfig](),
+ getMembers[BuildInfo](),
+ getMembers[SysEnvCC](),
+
+ getMembers[Functionality](),
+ getMembers[Author](),
+ getMembers[ComputationalRequirements](),
+ getMembers[ArgumentGroup](),
+ getMembers[Links](),
+ getMembers[References](),
+ getMembers[Scope](),
+
+ getMembers[Runner](),
+ getMembers[ExecutableRunner](),
+ getMembers[NextflowRunner](),
+
+ getMembers[Engine](),
+ getMembers[NativeEngine](),
+ getMembers[DockerEngine](),
+
+ getMembers[Platform](),
+ getMembers[NativePlatform](),
+ getMembers[DockerPlatform](),
+ getMembers[NextflowPlatform](),
+ )
}
-
- private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]) = {
- val annMembers = members
- .map{ case (memberName, memberInfo) => {
- val h = memberInfo.head
- val annotations = memberInfo.flatMap(_.symbol.annotations)
- (h.fullName, h.symbol.info.toString, annotations, h.className, h.inheritanceIndex, Nil)
- } }
- .filter(_._3.length > 0)
- val annThis = ("__this__", classes.head.name.toString(), classes.head.annotations, "", 0, classes.map(_.fullName))
- val allAnnotations = annThis :: annMembers.toList
- allAnnotations
- .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class
+ object memberData_part2 {
+ val part = List(
+ getMembers[Requirements](),
+ getMembers[ApkRequirements](),
+ getMembers[AptRequirements](),
+ getMembers[DockerRequirements](),
+ getMembers[JavaScriptRequirements](),
+ getMembers[PythonRequirements](),
+ getMembers[RRequirements](),
+ getMembers[RubyRequirements](),
+ getMembers[YumRequirements](),
+
+ getMembers[Argument[_]](),
+ getMembers[BooleanArgument](),
+ getMembers[BooleanTrueArgument](),
+ getMembers[BooleanFalseArgument](),
+ getMembers[DoubleArgument](),
+ getMembers[FileArgument](),
+ getMembers[IntegerArgument](),
+ getMembers[LongArgument](),
+ getMembers[StringArgument](),
+ )
}
-
- private val getSchema = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match {
- case (members, classes) => {
- annotationsOf(members, classes).map{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) }
- }
+ object memberData_part3 {
+ val part = List(
+ getMembers[Resource](),
+ getMembers[BashScript](),
+ getMembers[CSharpScript](),
+ getMembers[Executable](),
+ getMembers[JavaScriptScript](),
+ getMembers[NextflowScript](),
+ getMembers[PlainFile](),
+ getMembers[PythonScript](),
+ getMembers[RScript](),
+ getMembers[ScalaScript](),
+
+ getMembers[NextflowDirectives](),
+ getMembers[NextflowAuto](),
+ getMembers[NextflowConfig](),
+
+ getMembers[Dependency](),
+ getMembers[Repository](),
+ getMembers[LocalRepository](),
+ getMembers[GitRepository](),
+ getMembers[GithubRepository](),
+ getMembers[ViashhubRepository](),
+ getMembers[RepositoryWithName](),
+ getMembers[LocalRepositoryWithName](),
+ getMembers[GitRepositoryWithName](),
+ getMembers[GithubRepositoryWithName](),
+ getMembers[ViashhubRepositoryWithName](),
+ )
}
- // get all parameters for a given type, including parent class annotations
- def getParameters[T: TypeTag]() = getSchema(getMembers[T]())
+ val fullData = memberData_part1.part ++ memberData_part2.part ++ memberData_part3.part
// Main call for documentation output
- lazy val fullData: List[List[ParameterSchema]] = schemaClasses.map{ v => getSchema(v)}
lazy val data: List[List[ParameterSchema]] = fullData.map(_.filter(p => !p.hasUndocumented && !p.hasInternalFunctionality))
def getKeyFromParamList(data: List[ParameterSchema]): String = data.find(p => p.name == "__this__").get.`type`
def getJson: Json = data.asJson
- private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = {
- val issueMembers = members
- .toList
- .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement.
- .map{ case (k, v) => (k, v.map(_.symbol.annotations.length).sum) } // (name, # annotations)
- .filter(_._2 == 0)
- .map(_._1)
-
- val ownClassArr = if (classes.head.annotations.length == 0) Seq("__this__") else Nil
- issueMembers ++ ownClassArr
- }
-
- def getMemberName(members: Map[String,List[MemberInfo]], classes: List[Symbol]): String = classes.head.shortName
-
- // Main call for checking whether all arguments are annotated
+ // Main call for checking whether all arguments are annotated with a description
// Add extra non-annotated value so we can always somewhat check the code is functional
- def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap {
- v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _))
- }.toMap
+ def getAllNonAnnotated: List[(String, String)] = (data :+ getMembers[DeprecatedOrRemovedSchema]()).flatMap {
+ members => {
+ val notAnnonated = members.filter(p => p.description == None)
+ val thisType = members.find(p => p.name == "__this__").get.`type`
+ notAnnonated.map(p => (thisType, p.name))
+ }
+ }
def getAllDeprecations: Map[String, DeprecatedOrRemovedSchema] = {
val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap
diff --git a/src/main/scala/io/viash/schemas/JsonSchema.scala b/src/main/scala/io/viash/schemas/JsonSchema.scala
index 2f74a5570..0fb96dd99 100644
--- a/src/main/scala/io/viash/schemas/JsonSchema.scala
+++ b/src/main/scala/io/viash/schemas/JsonSchema.scala
@@ -200,14 +200,15 @@ object JsonSchema {
(p.name, mapType(s, pDescription))
case s if p.name == "type" && subclass.isDefined =>
+ var subclassString = subclass.get.stripSuffix("withname")
if (config.minimal) {
("type", Json.obj(
- "const" -> Json.fromString(subclass.get)
+ "const" -> Json.fromString(subclassString)
))
} else {
("type", Json.obj(
"description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class
- "const" -> Json.fromString(subclass.get)
+ "const" -> Json.fromString(subclassString)
))
}
@@ -286,7 +287,8 @@ object JsonSchema {
"DockerSetupStrategy" -> createEnum(DockerSetupStrategy.objs.map(obj => obj.id).toSeq, Some("The Docker setup strategy to use when building a container."), Some("TODO add descriptions to different strategies")),
"Direction" -> createEnum(Seq("input", "output"), Some("Makes this argument an `input` or an `output`, as in does the file/folder needs to be read or written. `input` by default."), None),
"Status" -> createEnum(Seq("enabled", "disabled", "deprecated"), Some("Allows setting a component to active, deprecated or disabled."), None),
- "DoubleStrings" -> createEnum(Seq("+infinity", "-infinity", "nan"), None, None)
+ "DoubleStrings" -> createEnum(Seq("+infinity", "-infinity", "nan"), None, None),
+ "ScopeEnum" -> createEnum(Seq("test", "private", "public"), Some("The scope of the component. `public` by default."), None),
)
} else {
Seq(
@@ -294,7 +296,8 @@ object JsonSchema {
"Direction" -> createEnum(Seq("input", "output"), Some("Makes this argument an `input` or an `output`, as in does the file/folder needs to be read or written. `input` by default."), None),
"Status" -> createEnum(Seq("enabled", "disabled", "deprecated"), Some("Allows setting a component to active, deprecated or disabled."), None),
"DockerResolveVolume" -> createEnum(Seq("manual", "automatic", "auto", "Manual", "Automatic", "Auto"), Some("Enables or disables automatic volume mapping. Enabled when set to `Automatic` or disabled when set to `Manual`. Default: `Automatic`"), Some("TODO make fully case insensitive")),
- "DoubleStrings" -> createEnum(Seq("+.inf", "+inf", "+infinity", "positiveinfinity", "positiveinf", "-.inf", "-inf", "-infinity", "negativeinfinity", "negativeinf", ".nan", "nan"), None, None)
+ "DoubleStrings" -> createEnum(Seq("+.inf", "+inf", "+infinity", "positiveinfinity", "positiveinf", "-.inf", "-inf", "-infinity", "negativeinfinity", "negativeinf", ".nan", "nan"), None, None),
+ "ScopeEnum" -> createEnum(Seq("test", "private", "public"), Some("The scope of the component. `public` by default."), None),
)
}
@@ -307,7 +310,7 @@ object JsonSchema {
}
Json.obj(
- "$schema" -> Json.fromString("https://json-schema.org/draft-07/schema#"),
+ "$schema" -> Json.fromString("http://json-schema.org/draft-07/schema#"),
"definitions" -> Json.obj(
definitions: _*
),
diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala
index ee4d847ca..de040d2c9 100644
--- a/src/main/scala/io/viash/schemas/ParameterSchema.scala
+++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala
@@ -17,8 +17,8 @@
package io.viash.schemas
-import scala.reflect.runtime.universe._
import io.viash.schemas.internalFunctionality
+import scala.annotation.Annotation
final case class ParameterSchema(
name: String,
@@ -39,48 +39,8 @@ final case class ParameterSchema(
)
object ParameterSchema {
- // Aid processing `augmentString` strings
- private def unfinishedStringStripMargin(s: String, marginChar: Char = '|'): String = {
- s.replaceAll("\\\\n", "\n").stripMargin(marginChar)
- }
-
- private def mapTreeList(l: List[Tree], marginChar: Char = '|'): String = {
- l.map(i => i match {
- case Literal(Constant(value: String)) =>
- unfinishedStringStripMargin(value, marginChar)
- case _ =>
- "unmatched in mapTreeList: " + i.toString()
- }).mkString
- }
-
- // Traverse tree information and extract values or lists of values
- private def annotationToStrings(ann: Annotation):(String, List[String]) = {
- val name = ann.tree.tpe.toString()
- val values = ann.tree match {
- case Apply(c, args: List[Tree]) =>
- args.collect({
- case i: Tree =>
- i match {
- // Here 'Apply' contains lists
- // While 'Select' has a single element
- case Literal(Constant(value: String)) =>
- value
- // case Select(Select(a, b), stripMargin) =>
- // unfinishedStringStripMargin(b)
- case Select(Apply(a, a2), b) if b.toString == "stripMargin" =>
- mapTreeList(a2)
- case Apply(Select(Apply(a, a2), b), stripMargin) if b.toString == "stripMargin" =>
- val stripper = stripMargin.head.toString.charAt(1)
- mapTreeList(a2, stripper)
- case _ =>
- "unmatched in annotationToStrings: " + i.toString()
- }
- })
- }
- (name, values)
- }
- def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): ParameterSchema = {
+ def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[(String, List[String])]): ParameterSchema = {
def beautifyTypeName(s: String): String = {
@@ -118,7 +78,6 @@ object ParameterSchema {
}
}
- val annStrings = annotations.map(annotationToStrings(_))
val hierarchyOption = hierarchy match {
case l if l.length > 0 => Some(l)
case _ => None
@@ -127,7 +86,7 @@ object ParameterSchema {
// name is e.g. "io.viash.config.Config.name", only keep "name"
// name can also be "__this__"
// Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'.
- val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head})
+ val nameOverride = annotations.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head})
val nameFromClass = name.split('.').last
val name_ = (nameOverride, nameFromClass) match {
case (Some(_), "__this__") => "__this__"
@@ -140,26 +99,26 @@ object ParameterSchema {
case (typeName, _, _) => typeName
}
- val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head})
- val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_))
- val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_))
+ val description = annotations.collectFirst({case (name, value) if name.endsWith("description") => value.head})
+ val example = annotations.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)).reverse
+ val exampleWithDescription = annotations.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)).reverse
val examples = example ::: exampleWithDescription match {
case l if l.length > 0 => Some(l)
case _ => None
}
- val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head})
- val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_))
- val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_))
- val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head})
+ val since = annotations.collectFirst({case (name, value) if name.endsWith("since") => value.head})
+ val deprecated = annotations.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_))
+ val removed = annotations.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_))
+ val defaultFromAnnotation = annotations.collectFirst({case (name, value) if name.endsWith("default") => value.head})
val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty")
val default = defaultFromAnnotation orElse defaultFromType
- val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match {
- case l if l.nonEmpty => Some(l)
+ val subclass = annotations.collect{ case (name, value) if name.endsWith("subclass") => value.head } match {
+ case l if l.nonEmpty => Some(l.sorted())
case _ => None
}
- val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")}
- val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")}
+ val undocumented = annotations.exists{ case (name, value) => name.endsWith("undocumented")}
+ val internalFunctionality = annotations.exists{ case (name, value) => name.endsWith("internalFunctionality")}
ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass, undocumented, internalFunctionality)
}
diff --git a/src/main/scala/io/viash/schemas/package.scala b/src/main/scala/io/viash/schemas/package.scala
index eb6210f5f..6011e3198 100644
--- a/src/main/scala/io/viash/schemas/package.scala
+++ b/src/main/scala/io/viash/schemas/package.scala
@@ -33,10 +33,10 @@ package object schemas {
class description(example: String) extends scala.annotation.StaticAnnotation
@getter @setter @beanGetter @beanSetter @field
- class deprecated(message: String, since: String, plannedRemoval: String) extends scala.annotation.StaticAnnotation
+ class deprecated(val message: String, val since: String, val plannedRemoval: String) extends scala.annotation.StaticAnnotation
@getter @setter @beanGetter @beanSetter @field
- class removed(message: String, deprecatedSince: String, since: String) extends scala.annotation.StaticAnnotation
+ class removed(val message: String, val deprecatedSince: String, val since: String) extends scala.annotation.StaticAnnotation
@getter @setter @beanGetter @beanSetter @field
class default(default: String) extends scala.annotation.StaticAnnotation
diff --git a/src/main/scala/io/viash/wrapper/BashWrapper.scala b/src/main/scala/io/viash/wrapper/BashWrapper.scala
index ee8b4b42c..1a3a400d3 100644
--- a/src/main/scala/io/viash/wrapper/BashWrapper.scala
+++ b/src/main/scala/io/viash/wrapper/BashWrapper.scala
@@ -25,6 +25,7 @@ import java.nio.file.Paths
import io.viash.ViashNamespace
import io.viash.config.arguments._
import io.viash.config.resources.Executable
+import io.viash.helpers.data_structures.oneOrMoreToList
object BashWrapper {
val metaArgs: List[Argument[_]] = {
@@ -138,6 +139,19 @@ object BashWrapper {
}
}
+ def generateHelp(helpSections: List[(String, String)]): String = {
+ val sections = helpSections.sortBy(_._1).map(_._2)
+ val helpStr = joinSections(sections).split("\n")
+ .map(h => Bash.escapeString(h, quote = true))
+ .mkString(" echo \"", "\"\n echo \"", "\"")
+ val functionStr =
+ s"""# ViashHelp: Display helpful explanation about this executable
+ |function ViashHelp {
+ |$helpStr
+ |}""".stripMargin
+ spaceCode(functionStr)
+ }
+
/**
* Joins multiple strings such that there are two spaces between them.
*
@@ -202,7 +216,7 @@ object BashWrapper {
// if we want to debug our code
case Some(res) if debugPath.isDefined =>
- val code = res.readWithInjection(argsMetaAndDeps, config).get
+ val code = res.readWithInjection(argsMetaAndDeps, config)
val escapedCode = Bash.escapeString(code, allowUnescape = true)
s"""
@@ -214,7 +228,7 @@ object BashWrapper {
// if mainResource is a script
case Some(res) =>
- val code = res.readWithInjection(argsMetaAndDeps, config).get
+ val code = res.readWithInjection(argsMetaAndDeps, config)
val escapedCode = Bash.escapeString(code, allowUnescape = true)
// check whether the script can be written to a temprorary location or
@@ -314,6 +328,7 @@ object BashWrapper {
|${spaceCode(allMods.preParse)}
|# preparse bashwrapper mods end --------------------------------
|
+ |${generateHelp(allMods.helpStrings)}
|# initialise array
|VIASH_POSITIONAL_ARGS=()
|
@@ -377,18 +392,8 @@ object BashWrapper {
private def generateHelp(config: Config) = {
- val help = Helper.generateHelp(config)
- val helpStr = help
- .map(h => Bash.escapeString(h, quote = true))
- .mkString(" echo \"", "\"\n echo \"", "\"")
-
- val preParse =
- s"""# ViashHelp: Display helpful explanation about this executable
- |function ViashHelp {
- |$helpStr
- |}""".stripMargin
-
- BashWrapperMods(preParse = preParse)
+ val help = Helper.generateHelp(config).mkString("\n")
+ BashWrapperMods(helpStrings = List(("", help)))
}
private def generateParsers(params: List[Argument[_]]) = {
@@ -636,13 +641,25 @@ object BashWrapper {
| ViashWarning '${param.name}' specifies a maximum value but the value was not verified as neither \\'bc\\' or \\'awk\\' are present on the system.
| fi
|""".stripMargin
- def minCheckInt(min: Long) =
+ def minCheckInt(min: Int) =
+ s""" if [[ $$${param.VIASH_PAR} -lt $min ]]; then
+ | ViashError '${param.name}' has be more than or equal to $min. Use "--help" to get more information on the parameters.
+ | exit 1
+ | fi
+ |""".stripMargin
+ def maxCheckInt(max: Int) =
+ s""" if [[ $$${param.VIASH_PAR} -gt $max ]]; then
+ | ViashError '${param.name}' has be less than or equal to $max. Use "--help" to get more information on the parameters.
+ | exit 1
+ | fi
+ |""".stripMargin
+ def minCheckLong(min: Long) =
s""" if [[ $$${param.VIASH_PAR} -lt $min ]]; then
| ViashError '${param.name}' has be more than or equal to $min. Use "--help" to get more information on the parameters.
| exit 1
| fi
|""".stripMargin
- def maxCheckInt(max: Long) =
+ def maxCheckLong(max: Long) =
s""" if [[ $$${param.VIASH_PAR} -gt $max ]]; then
| ViashError '${param.name}' has be less than or equal to $max. Use "--help" to get more information on the parameters.
| exit 1
@@ -651,13 +668,13 @@ object BashWrapper {
val minCheck = param match {
case p: IntegerArgument if min.isDefined => minCheckInt(min.get)
- case p: LongArgument if min.isDefined => minCheckInt(min.get)
+ case p: LongArgument if min.isDefined => minCheckLong(min.get)
case p: DoubleArgument if min.isDefined => minCheckDouble(min.get)
case _ => ""
}
val maxCheck = param match {
case p: IntegerArgument if max.isDefined => maxCheckInt(max.get)
- case p: LongArgument if max.isDefined => maxCheckInt(max.get)
+ case p: LongArgument if max.isDefined => maxCheckLong(max.get)
case p: DoubleArgument if max.isDefined => maxCheckDouble(max.get)
case _ => ""
}
@@ -783,6 +800,15 @@ object BashWrapper {
private def generateComputationalRequirements(config: Config) = {
+
+ val helpStrings =
+ """Viash built in Computational Requirements:
+ | ---cpus=INT
+ | Number of CPUs to use
+ | ---memory=STRING
+ | Amount of memory to use. Examples: 4GB, 3MiB.
+ |""".stripMargin
+
val compArgs = List(
("---cpus", "VIASH_META_CPUS", config.requirements.cpus.map(_.toString)),
("---memory", "VIASH_META_MEMORY", config.requirements.memoryAsBytes.map(_.toString + "b"))
@@ -861,6 +887,7 @@ object BashWrapper {
// return output
BashWrapperMods(
+ helpStrings = List(("Computational Requirements", helpStrings)),
parsers = parsers,
postParse = BashWrapper.joinSections(List(defaultsStrs, memoryCalculations))
)
diff --git a/src/main/scala/io/viash/wrapper/BashWrapperMods.scala b/src/main/scala/io/viash/wrapper/BashWrapperMods.scala
index 6d537eea9..37b4e4dcb 100644
--- a/src/main/scala/io/viash/wrapper/BashWrapperMods.scala
+++ b/src/main/scala/io/viash/wrapper/BashWrapperMods.scala
@@ -21,6 +21,7 @@ import io.viash.config.arguments.Argument
case class BashWrapperMods(
preParse: String = "",
+ helpStrings: List[(String, String)] = Nil,
parsers: String = "",
postParse: String = "",
preRun: String = "",
@@ -31,6 +32,7 @@ case class BashWrapperMods(
def `++`(other: BashWrapperMods): BashWrapperMods = {
BashWrapperMods(
preParse = BashWrapper.joinSections(List(preParse, other.preParse)),
+ helpStrings = helpStrings ++ other.helpStrings,
parsers = BashWrapper.joinSections(List(parsers, other.parsers), middle = "\n"),
postParse = BashWrapper.joinSections(List(postParse, other.postParse)),
preRun = BashWrapper.joinSections(List(preRun, other.preRun)),
diff --git a/src/test/resources/testnextflowvdsl3/src/multiple_emit_channels/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/multiple_emit_channels/config.vsh.yaml
new file mode 100644
index 000000000..77d26638b
--- /dev/null
+++ b/src/test/resources/testnextflowvdsl3/src/multiple_emit_channels/config.vsh.yaml
@@ -0,0 +1,31 @@
+name: multiple_emit_channels
+argument_groups:
+ - name: Outputs
+ arguments:
+ - name: "--input"
+ type: file
+ description: Input file
+ required: true
+ example: input.txt
+ - name: "--step_1_output"
+ required: true
+ type: file
+ direction: output
+ - name: "--step_3_output"
+ required: true
+ type: file
+ direction: output
+ - name: "--multiple_output"
+ required: true
+ multiple: true
+ type: file
+ direction: output
+resources:
+ - type: nextflow_script
+ path: main.nf
+ entrypoint: base
+dependencies:
+ - name: step1
+ - name: step3
+platforms:
+ - type: nextflow
diff --git a/src/test/resources/testnextflowvdsl3/src/multiple_emit_channels/main.nf b/src/test/resources/testnextflowvdsl3/src/multiple_emit_channels/main.nf
new file mode 100644
index 000000000..0b447d8ef
--- /dev/null
+++ b/src/test/resources/testnextflowvdsl3/src/multiple_emit_channels/main.nf
@@ -0,0 +1,63 @@
+workflow base {
+ take: input_ch
+ main:
+
+ step_1_ch = input_ch
+ // test fromstate and tostate with list[string]
+ | step1.run(
+ fromState: ["input"],
+ toState: { id, output, state -> ["step_1_output": output.output, "multiple_output": output.output] }
+ )
+
+ step_3_ch = input_ch
+ // test fromstate and tostate with map[string, string]
+ | step3.run(
+ fromState: ["input"],
+ toState: { id, output, state -> ["step_3_output": output.output, "multiple_output": output.output] }
+ )
+
+ emit:
+ step_1_ch
+ step_3_ch
+}
+
+workflow test_base {
+ // todo: fix how `test_base` is able to access the test resources
+ Channel.value([
+ "foo",
+ [
+ "input": file("${params.rootDir}/resources/lines3.txt")
+ ]
+ ])
+ | multiple_emit_channels
+ | toList()
+ | view { output_list ->
+ assert output_list.size() == 1 : "output channel should contain 1 event"
+
+ def event = output_list[0]
+ assert event.size() == 2 : "outputs should contain two elements; [id, state]"
+ def id = event[0]
+
+ // check id
+ assert id == "foo" : "id should be foo"
+
+ // check state
+ def state = event[1]
+ assert state instanceof Map : "state should be a map"
+ assert "step_1_output" in state : "state should contain key 'step_1_output'"
+ assert "step_3_output" in state : "state should contain key 'step_3_output'"
+
+ def step_1_output = state.step_1_output
+ assert step_1_output instanceof Path: "step_1_output should be a file"
+ assert step_1_output.toFile().exists() : "step_1_output file should exist"
+
+ def step_3_output = state.step_3_output
+ assert step_3_output instanceof Path: "step_3_output should be a file"
+ assert step_3_output.toFile().exists() : "step_3_output file should exist"
+
+ assert "multiple_output" in state : "state should contain 'multiple_output'"
+ def multiple_output = state.multiple_output
+ assert multiple_output instanceof List: "multiple_output should be a list"
+ assert multiple_output.size() == 2
+ }
+}
\ No newline at end of file
diff --git a/src/test/resources/testnextflowvdsl3/src/sub_workflow/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/sub_workflow/config.vsh.yaml
index 240367069..7348fbb25 100644
--- a/src/test/resources/testnextflowvdsl3/src/sub_workflow/config.vsh.yaml
+++ b/src/test/resources/testnextflowvdsl3/src/sub_workflow/config.vsh.yaml
@@ -6,6 +6,10 @@ arguments:
- name: "--output"
type: file
direction: output
+ - name: "--required_int"
+ type: integer
+ direction: output
+ required: true
resources:
- type: nextflow_script
path: main.nf
diff --git a/src/test/resources/testnextflowvdsl3/src/sub_workflow/main.nf b/src/test/resources/testnextflowvdsl3/src/sub_workflow/main.nf
index 03acb8b44..2ecba3fca 100644
--- a/src/test/resources/testnextflowvdsl3/src/sub_workflow/main.nf
+++ b/src/test/resources/testnextflowvdsl3/src/sub_workflow/main.nf
@@ -10,6 +10,10 @@ workflow base {
return newState
}
)
+ | map {id, state ->
+ def newState = state + ["required_int": 1]
+ [id, newState]
+ }
emit:
output_ch
diff --git a/src/test/resources/verification/check_config/config.vsh.yaml b/src/test/resources/verification/check_config/config.vsh.yaml
new file mode 100644
index 000000000..dc14e577f
--- /dev/null
+++ b/src/test/resources/verification/check_config/config.vsh.yaml
@@ -0,0 +1,22 @@
+name: check_config
+arguments:
+ - name: --data
+ alternatives: -d
+ type: file
+ direction: input
+ required: true
+ - name: --schema
+ alternatives: -s
+ type: file
+ direction: input
+ required: true
+resources:
+ - type: bash_script
+ text: ajv validate -s $par_schema $par_data
+engines:
+ - type: docker
+ image: node:20
+ setup:
+ - type: javascript
+ # npm: ajv-cli
+ npm: "@jirutka/ajv-cli@6.0.0-beta.5"
diff --git a/src/test/scala/io/viash/TestingAllComponentsSuite.scala b/src/test/scala/io/viash/TestingAllComponentsSuite.scala
index 6a64f8762..44a9053c5 100644
--- a/src/test/scala/io/viash/TestingAllComponentsSuite.scala
+++ b/src/test/scala/io/viash/TestingAllComponentsSuite.scala
@@ -5,6 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite
import io.viash.helpers.Logger
import org.scalatest.ParallelTestExecution
import io.viash.lenses.ConfigLenses
+import io.viash.config.{decodeConfig, encodeConfig}
class TestingAllComponentsSuite extends AnyFunSuite with ParallelTestExecution {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
index a3f117432..a2567bfcf 100644
--- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
+++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala
@@ -446,17 +446,38 @@ class MainBuildAuxiliaryDockerRequirementsR extends AbstractMainBuildAuxiliaryDo
assert(output.output.contains("/usr/local/lib/R/site-library/glue/R/glue doesn't exist."))
}
- test("setup; check for a descriptive message when .script contains a single quote", DockerTest) { f =>
+ test("setup; check .script contains a single quote", DockerTest) { f =>
val newConfigFilePath = deriveEngineConfig(Some("""[{ "type": "r", "script": "print('hello world')" }]"""), None, "r_script_single_quote")
- val testOutput = TestHelper.testMainException[ConfigParserException](
+ val testOutput = TestHelper.testMain(
+ "build",
+ "-o", tempFolStr,
+ "--setup", "build",
+ newConfigFilePath
+ )
+
+ println(s"testOutput: ${testOutput}")
+
+ assert(TestHelper.checkDockerImageExists(dockerTag))
+ assert(executableRequirementsFile.exists)
+ assert(executableRequirementsFile.canExecute)
+
+ assert(testOutput.exitCode == Some(0))
+ }
+
+ test("setup; check installing a missing package returns an error", DockerTest) { f =>
+ val newConfigFilePath = deriveEngineConfig(Some("""[{ "type": "r", "packages": ["non-existing-package"] }]"""), None, "r_non_existing_package")
+
+ val testOutput = TestHelper.testMain(
"build",
"-o", tempFolStr,
"--setup", "build",
newConfigFilePath
)
- assert(testOutput.exceptionText == Some("assertion failed: R requirement '.script' field contains a single quote ('). This is not allowed."))
+ assert(testOutput.exitCode == Some(1))
+ assert(testOutput.stdout.contains("Error: Failed to install 'non-existing-package' from CRAN"))
+ assert(testOutput.stdout.contains("ERROR: failed to solve"))
}
}
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala
index 97a5eb268..20e7e348c 100644
--- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala
+++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala
@@ -11,6 +11,7 @@ import io.viash.helpers.{IO, Exec, Logger}
import io.viash.TestHelper
import java.nio.file.Path
import scala.annotation.meta.param
+import io.viash.helpers.data_structures._
class MainBuildAuxiliaryNativeParameterCheck extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala
index 44e5e4ce4..1504aef14 100644
--- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala
+++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala
@@ -9,6 +9,7 @@ import io.viash.config.Config
import scala.io.Source
import io.viash.helpers.{IO, Exec, Logger}
import io.viash.TestHelper
+import io.viash.helpers.data_structures._
class MainBuildAuxiliaryNativeUnknownParameter extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala
index 3064e6955..3cea33235 100644
--- a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala
+++ b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala
@@ -6,7 +6,6 @@ import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
import java.nio.file.{Files, Paths, StandardCopyOption}
-import scala.reflect.io.Directory
import java.io.ByteArrayOutputStream
import io.viash.exceptions.ExitException
diff --git a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
index f7bf653a1..e04f52d94 100644
--- a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
+++ b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala
@@ -6,7 +6,6 @@ import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
import java.nio.file.{Files, Paths, StandardCopyOption}
-import scala.reflect.io.Directory
import io.viash.ConfigDeriver
class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfterAll {
@@ -78,7 +77,7 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
assert(md5sum.r.findFirstMatchIn(hash).isDefined, s"Calculated md5sum doesn't match the given md5sum for $name")
}
- Directory(tmpFolderResourceDestinationFolder).deleteRecursively()
+ IO.deleteRecursively(tmpFolderResourceDestinationFolder.toPath)
checkTempDirAndRemove(testOutput.stdout, true, "viash_test_auxiliary_resources")
}
@@ -109,30 +108,30 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte
* @param expectDirectoryExists expect the directory to be present or not
* @return
*/
- def checkTempDirAndRemove(testText: String, expectDirectoryExists: Boolean, folderName: String = "viash_test_testbash"): Unit = {
+ def checkTempDirAndRemove(testText: String, expectDirectoryExists: Boolean, testDirName: String = "viash_test_testbash"): Unit = {
// Get temporary directory
val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r
- val tempPath = testText.replaceAll("\n", "") match {
+ val tempPathStr = testText.replaceAll("\n", "") match {
case FolderRegex(path) => path
case _ => ""
}
- assert(tempPath.contains(s"${IO.tempDir}/$folderName"))
+ assert(tempPathStr.contains(s"${IO.tempDir}/$testDirName"))
- val tempFolder = new Directory(Paths.get(tempPath).toFile)
+ val tempPath = Paths.get(tempPathStr)
if (expectDirectoryExists) {
// Check temporary directory is still present
- assert(tempFolder.exists)
- assert(tempFolder.isDirectory)
+ assert(Files.exists(tempPath))
+ assert(Files.isDirectory(tempPath))
// Remove the temporary directory
- tempFolder.deleteRecursively()
+ IO.deleteRecursively(tempPath)
}
// folder should always have been removed at this stage
- assert(!tempFolder.exists)
+ assert(!Files.exists(tempPath))
}
override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/config/ConfigTest.scala b/src/test/scala/io/viash/config/ConfigTest.scala
index 3c0bcabaa..8c8a08c19 100644
--- a/src/test/scala/io/viash/config/ConfigTest.scala
+++ b/src/test/scala/io/viash/config/ConfigTest.scala
@@ -14,6 +14,7 @@ import io.viash.engines.NativeEngine
import io.viash.runners.ExecutableRunner
import io.viash.helpers.IO
import io.viash.helpers.status
+import io.viash.ConfigDeriver
class ConfigTest extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
@@ -22,6 +23,10 @@ class ConfigTest extends AnyFunSuite with BeforeAndAfterAll {
private val tempFolStr = temporaryFolder.toString
private val nsPath = getClass.getResource("/testns/").getPath
+
+ private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath
+ private val temporaryConfigFolder = IO.makeTemp(s"viash_${this.getClass.getName}_")
+ private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryConfigFolder)
val infoJson = Yaml("""
|foo:
@@ -131,7 +136,55 @@ class ConfigTest extends AnyFunSuite with BeforeAndAfterAll {
assert(configs.filter(_.status == Some(status.ParseError)).length == 2, "Expect 2 failed component")
}
- // TODO: expand functionality tests
+ test("Test default scope value") {
+ val newConfigFilePath = configDeriver.derive(Nil, "default_scope")
+ val newConfig = Config.read(newConfigFilePath)
+
+ assert(newConfig.scope.isRight)
+ val scope = newConfig.scope.toOption.get
+ assert(scope.image == ScopeEnum.Public)
+ assert(scope.target == ScopeEnum.Public)
+ }
+
+ test("Test public scope value") {
+ val newConfigFilePath = configDeriver.derive(""".scope := "public"""", "public_scope")
+ val newConfig = Config.read(newConfigFilePath)
+
+ assert(newConfig.scope.isRight)
+ val scope = newConfig.scope.toOption.get
+ assert(scope.image == ScopeEnum.Public)
+ assert(scope.target == ScopeEnum.Public)
+ }
+
+ test("Test private scope value") {
+ val newConfigFilePath = configDeriver.derive(""".scope := "private"""", "private_scope")
+ val newConfig = Config.read(newConfigFilePath)
+
+ assert(newConfig.scope.isRight)
+ val scope = newConfig.scope.toOption.get
+ assert(scope.image == ScopeEnum.Private)
+ assert(scope.target == ScopeEnum.Private)
+ }
+
+ test("Test test scope value") {
+ val newConfigFilePath = configDeriver.derive(""".scope := "test"""", "test_scope")
+ val newConfig = Config.read(newConfigFilePath)
+
+ assert(newConfig.scope.isRight)
+ val scope = newConfig.scope.toOption.get
+ assert(scope.image == ScopeEnum.Test)
+ assert(scope.target == ScopeEnum.Test)
+ }
+
+ test("Test scope value with different image and target") {
+ val newConfigFilePath = configDeriver.derive(""".scope := {image: "test", target: "private"}""", "custom_scope")
+ val newConfig = Config.read(newConfigFilePath)
+
+ assert(newConfig.scope.isRight)
+ val scope = newConfig.scope.toOption.get
+ assert(scope.image == ScopeEnum.Test)
+ assert(scope.target == ScopeEnum.Private)
+ }
override def afterAll(): Unit = {
IO.deleteRecursively(temporaryFolder)
diff --git a/src/test/scala/io/viash/config/arguments/MergingTest.scala b/src/test/scala/io/viash/config/arguments/MergingTest.scala
index dff7331da..ca98ddaca 100644
--- a/src/test/scala/io/viash/config/arguments/MergingTest.scala
+++ b/src/test/scala/io/viash/config/arguments/MergingTest.scala
@@ -5,6 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite
import io.viash.helpers.Logger
import io.viash.helpers.circe.Convert
import io.viash.config.Config
+import io.viash.config.decodeConfig
class MergingTest extends AnyFunSuite {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala b/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala
index a87be837d..934c651f5 100644
--- a/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala
+++ b/src/test/scala/io/viash/config/arguments/StringArgumentTest.scala
@@ -10,6 +10,7 @@ import io.circe.yaml.{parser => YamlParser}
import io.viash.helpers.circe._
import io.viash.helpers.data_structures._
import io.viash.helpers.Logger
+import io.viash.config.arguments.decodeStringArgument
class StringArgumentTest extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/config/dependencies/Repository.scala b/src/test/scala/io/viash/config/dependencies/Repository.scala
index 54930314c..843675e87 100644
--- a/src/test/scala/io/viash/config/dependencies/Repository.scala
+++ b/src/test/scala/io/viash/config/dependencies/Repository.scala
@@ -32,7 +32,7 @@ class RepositoryTest extends AnyFunSuite {
val viashhubRepo = repo.get.asInstanceOf[ViashhubRepository]
assert(viashhubRepo.repo == "viash-io/viash")
assert(viashhubRepo.tag == Some("v2.0.0"))
- assert(viashhubRepo.uri == "https://viash-hub.com/viash-io/viash.git")
+ assert(viashhubRepo.uri == "https://packages.viash-hub.com/viash-io/viash.git")
}
test("Repository.unapply: handles viashhub syntax with implicit vsh organization") {
@@ -42,7 +42,7 @@ class RepositoryTest extends AnyFunSuite {
val viashhubRepo = repo.get.asInstanceOf[ViashhubRepository]
assert(viashhubRepo.repo == "viash")
assert(viashhubRepo.tag == Some("v2.0.0"))
- assert(viashhubRepo.uri == "https://viash-hub.com/vsh/viash.git")
+ assert(viashhubRepo.uri == "https://packages.viash-hub.com/vsh/viash.git")
}
test("Repository.unapply: handles viashhub syntax with explicit vsh organization") {
@@ -52,7 +52,7 @@ class RepositoryTest extends AnyFunSuite {
val viashhubRepo = repo.get.asInstanceOf[ViashhubRepository]
assert(viashhubRepo.repo == "vsh/viash")
assert(viashhubRepo.tag == Some("v2.0.0"))
- assert(viashhubRepo.uri == "https://viash-hub.com/vsh/viash.git")
+ assert(viashhubRepo.uri == "https://packages.viash-hub.com/vsh/viash.git")
}
test("Repository.unapply: handles local syntax") {
diff --git a/src/test/scala/io/viash/e2e/build/DockerSuite.scala b/src/test/scala/io/viash/e2e/build/DockerSuite.scala
index 9b21ccbbd..90b60da8a 100644
--- a/src/test/scala/io/viash/e2e/build/DockerSuite.scala
+++ b/src/test/scala/io/viash/e2e/build/DockerSuite.scala
@@ -10,6 +10,7 @@ import io.viash.helpers.{IO, Exec, Logger}
import io.viash.config.Config
import scala.io.Source
+import io.viash.helpers.data_structures._
class DockerSuite extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/e2e/build/NativeSuite.scala b/src/test/scala/io/viash/e2e/build/NativeSuite.scala
index 49ada6588..b3a561379 100644
--- a/src/test/scala/io/viash/e2e/build/NativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/build/NativeSuite.scala
@@ -12,6 +12,7 @@ import scala.io.Source
import io.viash.helpers.{IO, Exec, Logger}
import io.viash.exceptions.ConfigParserException
import java.nio.file.Files
+import io.viash.helpers.data_structures._
class NativeSuite extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala
index 1ff1695b9..d697a3990 100644
--- a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala
+++ b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala
@@ -1,4 +1,4 @@
-package io.viash.e2e.export
+package io.viash.e2e.`export`
import io.viash._
import io.viash.helpers.Logger
@@ -171,7 +171,7 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
"export", "json_schema"
)
- assert(testOutput.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(testOutput.stdout.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(testOutput.stdout.contains("""- $ref: "#/definitions/Config""""))
}
@@ -180,7 +180,7 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
"export", "json_schema", "--format", "yaml"
)
- assert(testOutput.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(testOutput.stdout.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(testOutput.stdout.contains("""- $ref: "#/definitions/Config""""))
}
@@ -191,7 +191,7 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
)
val lines = helpers.IO.read(tempFile.toUri())
- assert(lines.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(lines.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(lines.contains("""- $ref: "#/definitions/Config""""))
}
@@ -202,7 +202,7 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
assert(testOutput.stdout.startsWith(
"""{
- | "$schema" : "https://json-schema.org/draft-07/schema#",
+ | "$schema" : "http://json-schema.org/draft-07/schema#",
| "definitions" : {
|""".stripMargin))
assert(testOutput.stdout.contains(""""$ref" : "#/definitions/Config""""))
@@ -217,7 +217,7 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
val lines = helpers.IO.read(tempFile.toUri())
assert(lines.startsWith(
"""{
- | "$schema" : "https://json-schema.org/draft-07/schema#",
+ | "$schema" : "http://json-schema.org/draft-07/schema#",
| "definitions" : {
|""".stripMargin))
assert(lines.contains(""""$ref" : "#/definitions/Config""""))
@@ -243,16 +243,16 @@ class MainExportSuite extends AnyFunSuite with BeforeAndAfter {
"--strict", "--minimal"
)
- assert(testOutput.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(testOutput.stdout.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(testOutput.stdout.contains("""- $ref: "#/definitions/Config""""))
- assert(testOutputStrict.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(testOutputStrict.stdout.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(testOutputStrict.stdout.contains("""- $ref: "#/definitions/Config""""))
- assert(testOutputMinimal.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(testOutputMinimal.stdout.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(testOutputMinimal.stdout.contains("""- $ref: "#/definitions/Config""""))
- assert(testOutputStrictMinimal.stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#""""))
+ assert(testOutputStrictMinimal.stdout.startsWith("""$schema: "http://json-schema.org/draft-07/schema#""""))
assert(testOutputStrictMinimal.stdout.contains("""- $ref: "#/definitions/Config""""))
// thresholds were chosen empirically
diff --git a/src/test/scala/io/viash/e2e/export/MainExportValidation.scala b/src/test/scala/io/viash/e2e/export/MainExportValidation.scala
new file mode 100644
index 000000000..bc1a4552b
--- /dev/null
+++ b/src/test/scala/io/viash/e2e/export/MainExportValidation.scala
@@ -0,0 +1,64 @@
+package io.viash.e2e.`export`
+
+import io.viash._
+import io.viash.helpers.{Logger, IO}
+import org.scalatest.funsuite.AnyFunSuite
+
+import java.nio.file.{Files, Path, Paths}
+import org.scalatest.BeforeAndAfterAll
+
+class MainExportValidation extends AnyFunSuite with BeforeAndAfterAll {
+ Logger.UseColorOverride.value = Some(false)
+ private val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_")
+
+ private val configFile = getClass.getResource("/testbash/config.vsh.yaml").getPath
+ private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryFolder)
+
+ private val schemaFile = temporaryFolder.resolve("schema.json")
+ private val validation = getClass.getResource("/verification/check_config/config.vsh.yaml").getPath
+
+ test("Export json schema") {
+ val testOutput = TestHelper.testMain(
+ "export", "json_schema",
+ "--format", "json",
+ "--output", schemaFile.toString()
+ )
+
+ assert(testOutput.exitCode == Some(0))
+ assert(Files.exists(schemaFile))
+ }
+
+ test("validate testbash with viash export json_schema", DockerTest) {
+ val newConfigFilePath = configDeriver.derive(
+ ".version := \"0.1\"",
+ "testbash_version_string"
+ )
+
+ val testOutput2 = TestHelper.testMain(
+ "run", validation.toString(),
+ "--",
+ "--schema", schemaFile.toString(),
+ "--data", newConfigFilePath
+ )
+
+ assert(testOutput2.exitCode == Some(0), s"Validation failed; $testOutput2")
+ assert(testOutput2.stdout.contains("testbash_version_string.vsh.yaml valid"))
+ }
+
+ test("validate testbash with viash export json_schema, invalid config - version should be a string", DockerTest) {
+ val testOutput2 = TestHelper.testMain(
+ "run", validation.toString(),
+ "--",
+ "--schema", schemaFile.toString(),
+ "--data", configFile
+ )
+
+ assert(testOutput2.exitCode == Some(1))
+ assert(testOutput2.stdout.contains("testbash/config.vsh.yaml invalid"))
+ assert(testOutput2.stdout.contains("^^^ must be string"))
+ }
+
+ override def afterAll(): Unit = {
+ IO.deleteRecursively(temporaryFolder)
+ }
+}
diff --git a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala
index a3c4a857d..4779023f2 100644
--- a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala
@@ -11,6 +11,7 @@ import java.io.File
import java.nio.file.Paths
import scala.io.Source
import java.io.ByteArrayOutputStream
+import io.viash.helpers.data_structures._
class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala
index 63fe346c0..6470400d6 100644
--- a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala
@@ -11,6 +11,7 @@ import org.scalatest.funsuite.AnyFunSuite
import java.io.File
import java.nio.file.Paths
import scala.io.Source
+import io.viash.config.decodeConfig
class MainNSListNativeSuite extends AnyFunSuite{
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala
index 43c6b74fb..a4650cf28 100644
--- a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala
+++ b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala
@@ -8,7 +8,6 @@ import io.viash.helpers.IO
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
-import scala.reflect.io.Directory
import sys.process._
class MainRunDockerSuite extends AnyFunSuite with BeforeAndAfterAll {
diff --git a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala
index 44ffd466b..2c33a0771 100644
--- a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala
+++ b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala
@@ -8,10 +8,9 @@ import io.viash.helpers.{IO, Exec, Logger}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
-import scala.reflect.io.Directory
import sys.process._
import org.scalatest.ParallelTestExecution
-import java.nio.file.Path
+import java.nio.file.{Path, Files}
class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with ParallelTestExecution{
Logger.UseColorOverride.value = Some(false)
@@ -336,26 +335,26 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall
// Get temporary directory
val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r
- val tempPath = testText.replaceAll("\n", "") match {
+ val tempPathStr = testText.replaceAll("\n", "") match {
case FolderRegex(path) => path
case _ => ""
}
- assert(tempPath.contains(s"${IO.tempDir}/viash_test_testbash"))
+ assert(tempPathStr.contains(s"${IO.tempDir}/viash_test_testbash"))
- val tempFolder = new Directory(Paths.get(tempPath).toFile)
+ val tempPath = Paths.get(tempPathStr)
if (expectDirectoryExists) {
// Check temporary directory is still present
- assert(tempFolder.exists)
- assert(tempFolder.isDirectory)
+ assert(Files.exists(tempPath))
+ assert(Files.isDirectory(tempPath))
// Remove the temporary directory
- tempFolder.deleteRecursively()
+ IO.deleteRecursively(tempPath)
}
// folder should always have been removed at this stage
- assert(!tempFolder.exists)
+ assert(!Files.exists(tempPath))
}
override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala
index 48422d054..771d5f4a5 100644
--- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala
+++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala
@@ -8,7 +8,6 @@ import io.viash.helpers.{IO, Exec, Logger}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
-import scala.reflect.io.Directory
import sys.process._
import io.viash.exceptions.ConfigParserException
import io.viash.exceptions.MissingResourceFileException
@@ -563,26 +562,26 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll {
// Get temporary directory
val FolderRegex = ".*Running tests in temporary directory: '([^']*)'.*".r
- val tempPath = testText.replaceAll("\n", "") match {
+ val tempPathStr = testText.replaceAll("\n", "") match {
case FolderRegex(path) => path
case _ => ""
}
- assert(tempPath.contains(s"${IO.tempDir}/$testDirName"))
+ assert(tempPathStr.contains(s"${IO.tempDir}/$testDirName"))
- val tempFolder = new Directory(Paths.get(tempPath).toFile)
+ val tempPath = Paths.get(tempPathStr)
if (expectDirectoryExists) {
// Check temporary directory is still present
- assert(tempFolder.exists)
- assert(tempFolder.isDirectory)
+ assert(Files.exists(tempPath))
+ assert(Files.isDirectory(tempPath))
// Remove the temporary directory
- tempFolder.deleteRecursively()
+ IO.deleteRecursively(tempPath)
}
// folder should always have been removed at this stage
- assert(!tempFolder.exists)
+ assert(!Files.exists(tempPath))
}
override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/escaping/EscapingNativeTest.scala b/src/test/scala/io/viash/escaping/EscapingNativeTest.scala
index 9f5544253..8c2ba48f3 100644
--- a/src/test/scala/io/viash/escaping/EscapingNativeTest.scala
+++ b/src/test/scala/io/viash/escaping/EscapingNativeTest.scala
@@ -9,6 +9,7 @@ import java.io.{IOException, UncheckedIOException}
import java.nio.file.{Files, Path, Paths}
import scala.io.Source
import io.viash.helpers.{IO, Exec, Logger}
+import io.viash.helpers.data_structures._
class EscapingNativeTest extends AnyFunSuite with BeforeAndAfterAll {
Logger.UseColorOverride.value = Some(false)
diff --git a/src/test/scala/io/viash/helpers/circe/Convert.scala b/src/test/scala/io/viash/helpers/circe/Convert.scala
index 8a00dc324..46621d750 100644
--- a/src/test/scala/io/viash/helpers/circe/Convert.scala
+++ b/src/test/scala/io/viash/helpers/circe/Convert.scala
@@ -5,12 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite
import io.circe._
import io.viash.exceptions.{ConfigYamlException, ConfigParserException}
-import shapeless.Lazy
-import scala.reflect.runtime.universe._
-
import io.circe.Decoder
-import io.circe.generic.extras.decoding.ConfiguredDecoder
-import io.circe.generic.extras.semiauto.deriveConfiguredDecoder
import io.viash.helpers.Logger
class ConvertTest extends AnyFunSuite with BeforeAndAfterAll {
diff --git a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala
index 60548d687..d45de5844 100644
--- a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala
+++ b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala
@@ -4,7 +4,6 @@ import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
import io.circe._
import io.circe.yaml.parser
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.viash.helpers.Logger
class ParseEitherTest extends AnyFunSuite with BeforeAndAfterAll {
diff --git a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala
index 763a01c3c..8c58a95cf 100644
--- a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala
+++ b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala
@@ -5,7 +5,6 @@ import org.scalatest.funsuite.AnyFunSuite
import io.circe._
import io.circe.yaml.parser
import io.viash.helpers.circe._
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.viash.helpers.data_structures._
import io.viash.helpers.Logger
diff --git a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala
index 063b6c7cd..28791e04e 100644
--- a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala
+++ b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala
@@ -4,7 +4,6 @@ import org.scalatest.BeforeAndAfterAll
import org.scalatest.funsuite.AnyFunSuite
import io.circe._
import io.circe.yaml.parser
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.viash.helpers.Logger
class ParseStringLikeTest extends AnyFunSuite with BeforeAndAfterAll {
diff --git a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala
index 182ef9a59..126338d20 100644
--- a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala
+++ b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala
@@ -5,7 +5,6 @@ import org.scalatest.funsuite.AnyFunSuite
import io.circe._
import io.circe.yaml.parser
import io.viash.helpers.circe._
-import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder}
import io.viash.helpers.data_structures._
import io.viash.helpers.Logger
import java.io.ByteArrayOutputStream
diff --git a/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala b/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala
index 9194df96b..2e0a770cb 100644
--- a/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/NextflowScriptTest.scala
@@ -200,6 +200,37 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll {
assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr")
}
+
+ test("Run multiple output channels standalone", NextflowTest) {
+ val (exitCode, stdOut, stdErr) = NextflowTestHelper.run(
+ mainScript = "target/nextflow/multiple_emit_channels/main.nf",
+ args = List(
+ "--id", "foo",
+ "--input", "resources/lines5.txt",
+ "--publish_dir", "output"
+ ),
+ cwd = tempFolFile
+ )
+
+ assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr")
+ }
+
+
+ test("Run multiple output channels check output", DockerTest, NextflowTest) {
+ val (exitCode, stdOut, stdErr) = NextflowTestHelper.run(
+ mainScript = "target/nextflow/multiple_emit_channels/main.nf",
+ entry = Some("test_base"),
+ args = List(
+ "--rootDir", tempFolStr,
+ "--publish_dir", "output"
+ ),
+ cwd = tempFolFile
+ )
+
+ assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr")
+ }
+
+
test("Check whether --help is same as Viash's --help", NextflowTest) {
// except that WorkflowHelper.nf will not print alternatives, and
// will always prefix argument names with -- (so --foo, not -f or foo).
diff --git a/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala b/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala
index 0c5e75b73..6565630e6 100644
--- a/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala
+++ b/src/test/scala/io/viash/runners/nextflow/NextflowTestHelper.scala
@@ -22,7 +22,7 @@ object NextflowTestHelper {
val lines = output.split("\n").find(DebugRegex.findFirstIn(_).isDefined)
assert(lines.isDefined)
- val DebugRegex(path) = lines.get
+ val DebugRegex(path) = lines.get : @unchecked
val src = Source.fromFile(path)
try {
diff --git a/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala b/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala
index c048348c7..e8188119a 100644
--- a/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/Vdsl3ModuleTest.scala
@@ -105,11 +105,16 @@ class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll {
"run", workflowsPath + "/pipeline3/config.vsh.yaml",
"--", "--help"
)
+
+ // explicitly remove triple dash parameters
+ // these make sense when running from command line, not when running in nextflow
+ val correctedTestOutput = testOutput.stdout.replaceAll("""\n\nViash built in .*(\n\s{4}---.*\n(\s{8}.*)+)*""", "")
+
assert(testOutput.exitCode == Some(0))
// check if they are the same
- assert(correctedStdOut2 == testOutput.stdout)
+ assert(correctedStdOut2 == correctedTestOutput)
}
override def afterAll(): Unit = {
diff --git a/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala b/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala
index dba4be446..6dc39b365 100644
--- a/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala
+++ b/src/test/scala/io/viash/runners/nextflow/Vdsl3StandaloneTest.scala
@@ -94,6 +94,37 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll {
}
}
+ test("With output id and key keywords", NextflowTest) {
+ val (exitCode, stdOut, stdErr) = NextflowTestHelper.run(
+ mainScript = "target/nextflow/step2/main.nf",
+ args = List(
+ "--id", "foo",
+ "--input1", "resources/lines3.txt",
+ "--input2", "resources/lines5.txt",
+ "--output1", "$id.${id}.$key.${key}.txt",
+ // "--output2", "$foo$bar.txt", // can't do this (yet) because of Nextflow doesn't support '$' yet.
+ "--publish_dir", "moduleOutput2"
+ ),
+ cwd = tempFolFile
+ )
+
+ assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr")
+ assert(Files.exists(Paths.get(tempFolStr + "/moduleOutput2/foo.foo.step2.step2.txt")))
+ // assert(Files.exists(Paths.get(tempFolStr + "/moduleOutput2/$foo$bar.txt")))
+
+ val src1 = Source.fromFile(tempFolStr + "/moduleOutput2/foo.foo.step2.step2.txt")
+ // val src2 = Source.fromFile(tempFolStr + "/moduleOutput2/$foo$bar.txt")
+ try {
+ val moduleOut1 = src1.getLines().mkString(",")
+ // val moduleOut2 = src2.getLines().mkString(",")
+ assert(moduleOut1.equals("one,two,three"))
+ // assert(moduleOut2.equals("1,2,3,4,5"))
+ } finally {
+ src1.close()
+ // src2.close()
+ }
+ }
+
test("With yamlblob param_list", NextflowTest) {
val paramListStr = "[{input1: resources/lines3.txt, input2: resources/lines5.txt}]"
val (exitCode, stdOut, stdErr) = NextflowTestHelper.run(
diff --git a/src/test/scala/io/viash/schema/SchemaTest.scala b/src/test/scala/io/viash/schema/SchemaTest.scala
index 6fbbbc4fd..2f13f77b8 100644
--- a/src/test/scala/io/viash/schema/SchemaTest.scala
+++ b/src/test/scala/io/viash/schema/SchemaTest.scala
@@ -9,38 +9,18 @@ import io.viash.helpers.Logger
class SchemaTest extends AnyFunSuite with BeforeAndAfterAll with PrivateMethodTester{
Logger.UseColorOverride.value = Some(false)
-
- test("Check type name trimming") {
- val checks = Map (
- "foo" -> "foo",
- "foo.bar" -> "bar",
- "foo.bar.baz" -> "baz",
- "foo.bar[baz]" -> "bar[baz]",
- "foo[bar.baz]" -> "foo[baz]",
- "foo[bar,baz]" -> "foo[bar,baz]",
- "foo[bar.baz,quux]" -> "foo[baz,quux]",
- "foo[bar,baz.quux]" -> "foo[bar,quux]"
- )
-
- val trimTypeName = PrivateMethod[String](Symbol("trimTypeName"))
-
- for ((k, v) <- checks) {
- val res = CollectedSchemas invokePrivate trimTypeName(k)
- assert(res == v, s"$k -> $v != $res")
- }
- }
-
+
test("All schema class val members should be annotated") {
val nonAnnotated = CollectedSchemas.getAllNonAnnotated
- assert(nonAnnotated.contains("CollectedSchemas"))
- assert(nonAnnotated("CollectedSchemas") == "__this__")
+ assert(nonAnnotated.contains(("DeprecatedOrRemovedSchema", "__this__")))
- nonAnnotated.removed("CollectedSchemas").foreach {
- case (key, member) => Console.err.println(s"$key - $member")
+ nonAnnotated.foreach {
+ case (key, member) if key != "DeprecatedOrRemovedSchema" => Console.err.println(s"$key - $member")
+ case _ => ()
}
- assert(nonAnnotated.size == 1)
+ assert(nonAnnotated.size == 4) // DeprecatedOrRemovedSchema has 3 members, all of them unannotated + 1 __this__ member
}
test("Check formatting of deprecation annotations") {