diff --git a/.github/workflows/sbt_test.yml b/.github/workflows/sbt_test.yml index 7b71656ce..76b2f6dd8 100644 --- a/.github/workflows/sbt_test.yml +++ b/.github/workflows/sbt_test.yml @@ -11,14 +11,21 @@ jobs: fail-fast: false matrix: config: - - {name: 'ubuntu_latest', os: ubuntu-latest } - - {name: 'macos_latest', os: macos-latest } + - { name: 'ubuntu_latest', os: ubuntu-latest } + - { name: 'macos_latest', os: macos-latest } + java: + - { ver: '11', run_nextflow: true, run_coverage: false } + - { ver: '17', run_nextflow: true, run_coverage: true } + - { ver: '20', run_nextflow: false, run_coverage: false } steps: - uses: actions/checkout@v3 - - name: Set up Nextflow + - uses: viash-io/viash-actions/update-docker-engine@v3 if: runner.os == 'Linux' + + - name: Set up Nextflow + if: ${{ runner.os == 'Linux' && matrix.java.run_nextflow }} run: | mkdir -p "$HOME/.local/bin" echo "$HOME/.local/bin" >> $GITHUB_PATH @@ -42,7 +49,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: temurin - java-version: '11' + java-version: ${{ matrix.java.ver }} - name: Set up Scala run: | @@ -52,6 +59,7 @@ jobs: sudo apt-get update sudo apt-get install -y scala fi + - name: Set up Python uses: actions/setup-python@v4 with: @@ -59,12 +67,14 @@ jobs: - name: Run tests run: | - if [[ "${{ matrix.config.name }}" == ^ubuntu_latest$ ]]; then + if [[ "${{ matrix.config.name }}" =~ ^ubuntu.*$ ]] && [[ "${{ matrix.java.run_coverage }}" == "true" ]]; then # only run coverage on main runner sbt clean coverage test coverageReport + elif [[ "${{ matrix.config.name }}" =~ ^ubuntu.*$ ]] && [[ "${{ matrix.java.run_nextflow }}" == "false" ]]; then + sbt 'testOnly -- -l io.viash.NextflowTest' elif [[ "${{ matrix.config.os }}" =~ ^macos.*$ ]]; then # macOS on github actions does not have Docker, so skip those - sbt 'testOnly -- -l io.viash.DockerTest -l io.viash.NextFlowTest' + sbt 'testOnly -- -l io.viash.DockerTest -l io.viash.NextflowTest' else sbt test fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 40a82dfd0..82099070f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,89 @@ +# Viash 0.7.5 (2023-08-11): Minor breaking changes and new features + +This release contains minor breaking change due to deprecated or outdated functionality being removed. + +New functionality includes: + + - Export a JSON schema for the Viash config with `viash export json_schema` + + - Export a Bash or Zsh autocomplete script with `viash export cli_autocomplete` + + - Nextflow VDSL3 modules now have a `fromState` and `toState` argument to allow for better control of the data that gets passed to the module and how the state is managed in a Nextflow workflow. + +## BREAKING CHANGES + +* `viash export cli_schema`: Added `--format yaml/json` argument, default format is now a YAML (PR #448). + +* `viash export config_schema`: Added `--format yaml/json` argument, default format is now a YAML (PR #448). + +* `NextflowLegacyPlatform`: Removed deprecated code (PR #469). + +* `viash_*`: Remove legacy viash_build, viash_test and viash_push components (PR #470). + +* `ComputationalRequirements`, `Functionality`, `DockerPlatform`, `DockerRequirements`: Remove documentation of removed fields (PR #477). + +## NEW FUNCTIONALITY + +* `viash export json_schema`: Export a json schema derived from the class reflections and annotations already used by the `config_schema` (PR #446). + +* `viash export config_schema`: Output `default` values of member fields (PR #446). + +* `CI`: Test support for different Java versions on GitHub Actions (PR #456). Focussing on LTS releases starting from 11, so this is 11 and 17. Also test latest Java version, currently 20. + +* `viash test` and `viash ns test`: add `--setup` argument to determine the docker build strategy before a component is tested (PR #451). + +* `viash export cli_autocomplete`: Export a Bash or Zsh autocomplete script (PR #465 & #482). + +* `help message`: Print the relevant help message of (sub-)command when `--help` is given as an argument instead of only printing the help message when it is the leading argument and otherwise silently disregarding it (initially added in PR #472, replaced by PR #496). This is a new feature implemented in Scallop 5.0.0. + +* `Logging`: Add a Logger helper class (PR #485 & #490). Allows manually enabling or disabling colorizing TTY output by using `--colorize`. Add provisions for adding debugging or trace code which is not outputted by default. Changing logging level can be changed with `--loglevel`. These CLI arguments are currently hidden. + +* `NextflowPlatform`: Nextflow VDSL3 modules now have a `fromState` and `toState` argument to allow for better control of the data that gets passed to the module and how the state is managed in a Nextflow workflow (#479, PR #501). + +## MINOR CHANGES + +* `PythonScript`: Pass `-B` to Python to avoid creating `*.pyc` and `*.pyo` files on importing helper functions (PR #442). + +* `viash config`: Special double values now support `+.inf`, `-.inf` or `.nan` values (PR #446 and PR #450). The stringified versions `"+.inf"`, `"-.inf"` or `".nan"` are supported as well. This is in line with the yaml spec. + +* `system environment variables`: Add wrapper around `sys.env` and provide access to specific variables (PR #457). Has advantages for documentation output and testbenches. + +* `testbench`: Added some minor testbenches to tackle missing coverage (PR #459, #486, #488, #489, #492 & #494). + +* `viash export config_schema`: Simplify file structure (PR #464). + +* `helpers.Format`: Add a helper for the Format helper object (PR #466). + +* `testbench`: Use config deriver to create config variants for testing (PR #498). This reduces the amount of config files that need to be maintained. + +## BUG FIXES + +* `viash config`: Validate Viash config Yaml files better and try to give a more informative error message back to the user instead of a stack trace (PR #443). + +* `viash ns build`: Fix the error summary when a setup or push failure occurs. These conditions were not displayed and could cause confusion (PR #447). + +* `testbench`: Fix the viash version switch test bench not working for newer Java versions (PR #452). + +* `malformed input exception`: Capture MalformedInputExceptions when thrown by reading files with invalid Ascii characters when unsupported by Java (PR #458). + +* `viash project file parsing`: Give a more informative message when the viash project file fails to parse correctly (PR #475). + +* `DockerPlatform`: Fix issue when mounting an input or output folder containing spaces (PR #484). + +* `Config mod`: Fix a config mod where the filter should execute multiple deletes (PR #503). + +## DOCUMENTATION + +* `NextflowPlatform`: Add documentation for the usage and arguments of a VDSL3 module (PR #501). + +## INTERNAL CHANGES + +* `NextflowVDSL3Platform`: Renamed to `NextflowPlatform` (PR #469). + +* Rename mentions of `NextFlow` to `Nextflow` (PR #476). + +* `Reference static pages`: Move `.qmd` files from the website to a local folder here; `docs/reference` (PR #504). This way we can track behaviour changes that need to be documented locally. + # Viash 0.7.4 (2023-05-31): Minor bug fixes and minor improvements to VDSL3 Some small fixes and consistency improvements. diff --git a/build.sbt b/build.sbt index 162e480b8..6b200ca15 100644 --- a/build.sbt +++ b/build.sbt @@ -1,13 +1,13 @@ name := "viash" -version := "0.7.4" +version := "0.7.5" scalaVersion := "2.13.10" libraryDependencies ++= Seq( "org.scalactic" %% "scalactic" % "3.2.15" % "test", "org.scalatest" %% "scalatest" % "3.2.15" % "test", - "org.rogach" %% "scallop" % "4.1.0", + "org.rogach" %% "scallop" % "5.0.0", "org.scala-lang" % "scala-reflect" % scalaVersion.value, "org.scala-lang.modules" %% "scala-parser-combinators" % "2.1.1", "org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.4" diff --git a/docs/reference/cli/index.qmd b/docs/reference/cli/index.qmd new file mode 100644 index 000000000..7cf101221 --- /dev/null +++ b/docs/reference/cli/index.qmd @@ -0,0 +1,13 @@ +--- +title: CLI +listing: + - id: cli + contents: . + template: template.ejs +order: 10 +--- + +These are the available commands available on the Command Line Interface: + +:::{#cli} +::: \ No newline at end of file diff --git a/docs/reference/cli/template.ejs b/docs/reference/cli/template.ejs new file mode 100644 index 000000000..de662d38f --- /dev/null +++ b/docs/reference/cli/template.ejs @@ -0,0 +1,9 @@ +```{=html} + +``` \ No newline at end of file diff --git a/docs/reference/config_mods/index.qmd b/docs/reference/config_mods/index.qmd new file mode 100644 index 000000000..594cd2ff6 --- /dev/null +++ b/docs/reference/config_mods/index.qmd @@ -0,0 +1,49 @@ +--- +title: Dynamic Config Modding +order: 50 +--- + +Viash can modify a [viash config](/reference/config/index.html) at runtime using a custom Domain Specific Language (DSL). This allows making dynamic changes to your components or projects. +All Viash subcommands have support for the DSL through the `-c|--config_mod` parameter. The format for these is as follows: + +```bash +viash COMMAND -c '.SECTION.PROPERTY := VALUE' +``` + +Multiple config mods can be added by adding more `-c|--config_mod` parameters: + +```bash +viash COMMAND \ + -c '.SECTION.PROPERTY := VALUE' \ + -c '.SECTION.PROPERTY := VALUE' +``` + +## Examples + +Change the version of a component: + +```bash +viash build -c '.functionality.version := "0.3.0"' +``` + +Change the registry of a docker container: + +```bash +viash build -c \ + '.platforms[.type == "docker"].container_registry := "url-to-registry"' +``` + +Add an author to the list: + +```bash +viash build -c '.functionality.authors += { name: "Mr. T", role: "sponsor" }' +``` + +You can use dynamic config modding to alter the config of multiple components at once: + +```bash +viash ns build \ + -c '.functionality.version := "0.3.0"' \ + -c '.platforms[.type == "docker"].container_registry := "url-to-registry"' \ + -c '.functionality.authors += { name: "Mr. T", role: "sponsor" }' +``` \ No newline at end of file diff --git a/docs/reference/nextflow_vdsl3/import_module.qmd b/docs/reference/nextflow_vdsl3/import_module.qmd new file mode 100644 index 000000000..64f84b09e --- /dev/null +++ b/docs/reference/nextflow_vdsl3/import_module.qmd @@ -0,0 +1,107 @@ +--- +title: Import a VDSL3 module +--- + +A VDSL3 module is a Nextflow module generated by Viash. See the [guide](/guide/nextflow_vdsl3/introduction.qmd) for a more in-depth explanation on how to create Nextflow workflows with VDSL3 modules. + +## Importing a VDSL3 module + + +After building a VDSL3 module from a component, the VDSL3 module can be imported just like any other Nextflow module. + +**Example:** + +```groovy +include { mymodule } from 'target/nextflow/mymodule/main.nf' +``` + +## VDSL3 module interface + +VDSL3 modules are actually workflows which take one channel and emit one channel. It expects the channel events to be tuples containing an 'id' and a 'state': `[id, state]`, where `id` is a unique String and `state` is a `Map[String, Object]`. The resulting channel then consists of tuples `[id, new_state]`. + +**Example:** + +```groovy +workflow { + Channel.fromList([ + ["myid", [input: file("in.txt")]] + ]) + | mymodule +} +``` + +:::{.callout-note} +If the input tuple has more than two elements, the elements after the second element are passed through to the output tuple. +That is, an input tuple `[id, input, ...]` will result in a tuple `[id, output, ...]` after running the module. +For example, an input tuple `["foo", [input: file("in.txt")], "bar"]` will result in an output tuple `["foo", [output: file("out.txt")], "bar"]`. +::: + +## Customizing VDSL3 modules on the fly + +Usually, Nextflow processes are quite static objects. For example, changing its directives can be quite tricky. + +The `.run()` function is a unique feature for every VDSL3 module which allows dynamically altering the behaviour of a module from within the pipeline. For example, we use it to set the `publishDir` directive to `"output/"` so the output of that step in the pipeline will be stored as output. + +**Example:** + +```groovy +workflow { + Channel.fromList([ + ["myid", [input: file("in.txt")]] + ]) + | mymodule.run( + args: [k: 10], + directives: [cpus: 4, memory: "16 GB"] + ) +} +``` + +### Arguments of `.run()` + +- `key` (`String`): A unique key used to trace the process and help make names of output files unique. Default: the name of the Viash component. + +- `args` (`Map[String, Object]`): Argument overrides to be passed to the module. + +- `directives` (`Map[String, Object]`): Custom directives overrides. See the Nextflow documentation for a list of available directives. + +- `auto` (`Map[String, Boolean]`): Whether to apply certain automated processing steps. Default values are inherited from the [Viash config](/reference/config/platforms/nextflow/auto.qmd). + +- `auto.simplifyInput`: If `true`, if the input tuple is a single file and if the module only has a single input file, the input file will be passed the module accordingly. Default: `true` (inherited from Viash config). + +- `auto.simplifyOutput`: If `true`, if the output tuple is a single file and if the module only has a single output file, the output map will be transformed into a single file. Default: `true` (inherited from Viash config). + +- `auto.publish`: If `true`, the output files will be published to the `params.publishDir` folder. Default: `false` (inherited from Viash config). + +- `auto.transcript`: If `true`, the module's transcript will be published to the `params.transcriptDir` folder. Default: `false` (inherited from Viash config). + +- `map` (`Function`): Apply a map over the incoming tuple. Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }`. Default: `null`. + +- `mapId` (`Function`): Apply a map over the ID element of a tuple (i.e. the first element). Example: `{ id -> id + "_foo" }`. Default: `null`. + +- `mapData` (`Function`): Apply a map over the data element of a tuple (i.e. the second element). Example: `{ data -> [ input: data.output ] }`. Default: `null`. + +- `mapPassthrough` (`Function`): Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements). Example: `{ pt -> pt.drop(1) }`. Default: `null`. + +- `filter` (`Function`): Filter the channel. Example: `{ tup -> tup[0] == "foo" }`. Default: `null`. + +- `fromState`: Fetch data from the state and pass it to the module without altering the current state. `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + + - If it is `null`, the state will be passed to the module as is. + - If it is a `List[String]`, the data will be the values of the state at the given keys. + - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + + Example: `{ id, state -> [input: state.fastq_file] }` + Default: `null` + +- `toState`: Determine how the state should be updated after the module has been run. `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + + - If it is `null`, the state will be replaced with the output of the module. + - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + + Example: `{ id, output, state -> state + [counts: state.output] }` + Default: `{ id, output, state -> output }` + +- `debug`: Whether or not to print debug messages. Default: `false`. diff --git a/docs/reference/nextflow_vdsl3/index.qmd b/docs/reference/nextflow_vdsl3/index.qmd new file mode 100644 index 000000000..c33c9a067 --- /dev/null +++ b/docs/reference/nextflow_vdsl3/index.qmd @@ -0,0 +1,10 @@ +--- +title: Nextflow VDSL3 +order: 35 +--- + +Viash supports creating Nextflow workflows in multiple ways. + +* [Run a module as a standaline pipeline](run_module.qmd) +* [Import a VDSL3 module](import_module.qmd) +* Create a Nextflow workflow with dependencies \ No newline at end of file diff --git a/docs/reference/nextflow_vdsl3/run_module.qmd b/docs/reference/nextflow_vdsl3/run_module.qmd new file mode 100644 index 000000000..6d33adae2 --- /dev/null +++ b/docs/reference/nextflow_vdsl3/run_module.qmd @@ -0,0 +1,61 @@ +--- +title: Run a VDSL3 module +--- + + +Unlike typical Nextflow modules, VDSL3 modules can actually be used as a standalone pipeline. + +To run a VDSL3 module as a standalone pipeline, you need to specify the input parameters and a `--publish_dir` parameter, as Nextflow will automatically choose the parameter names of the output files. + +## Viewing the help message + +More information regarding a modules arguments can be shown by passing the `--help` parameter. + +**Example:** + +```bash +nextflow run target/nextflow/mycomponent/main.nf --help +``` + + +## Running a module as a standalone pipeline +You can run the executable by providing a value for each of the required arguments and `--publish_dir` (where output files are published). + +**Example:** + +```bash +nextflow run target/nextflow/mycomponent/main.nf \ + --input config.vsh.yaml \ + --publish_dir output/ +``` + + +## Passing a parameter list + +Every VDSL3 can accept a list of parameters to populate a Nextflow channel with. Assuming we want to process a set of input files in parallel, we can create a yaml file `params.yaml` containing the following information. + + +```yaml +param_list: + - id: sample1 + input: data/sample1.txt + - id: sample2 + input: data/sample2.txt + - id: sample3 + input: data/sample3.txt + - id: sample4 + input: data/sample4.txt +arg1: 10 +arg2: 5 +``` + +You can run the pipeline on the list of parameters using the `-params-file` parameter. + +```{bash} +nextflow run target/main.nf -params-file params.yaml --publish_dir output2 +``` + + +:::{.callout-tip} +You can also pass a YAML, CSV or JSON file to the `param_list` parameter. +::: \ No newline at end of file diff --git a/docs/reference/viash_code_block/index.qmd b/docs/reference/viash_code_block/index.qmd new file mode 100644 index 000000000..ee112adb7 --- /dev/null +++ b/docs/reference/viash_code_block/index.qmd @@ -0,0 +1,275 @@ +--- +title: Viash Code Block +order: 40 +--- + +{{< include ../../_includes/_language_chooser.qmd >}} + +**Example:** + +```{r setup, include=FALSE} +repo_path <- system("git rev-parse --show-toplevel", intern = TRUE) +source(paste0(repo_path, "/_includes/_r_helper.R")) +source(paste0(repo_path, "/guide/component/_language_examples.R")) +# escape languages +langs <- langs %>% + mutate(label = gsub("#", "\\\\#", label)) +``` + + +When running a Viash component with `viash run`, Viash will wrap your script into a Bash executable. In doing so, it strips away the "Viash placeholder" code block and replaces it by a bit of code to your script for reading any parameter values at runtime. + +## Recognizing the Viash placeholder code block + +```{r setup-config-inject, include=FALSE} +temp_dir <- tempfile("config_inject") +dir.create(temp_dir, recursive = TRUE, showWarnings = FALSE) +on.exit(unlink(temp_dir, recursive = TRUE), add = TRUE) +langs <- langs %>% + mutate( + config_path = paste0(temp_dir, "/", id, "/", basename(example_config)), + script_path = paste0(temp_dir, "/", id, "/", basename(example_script)) + ) +pwalk(langs, function(id, label, example_config, example_script, config_path, script_path, ...) { + dir.create(paste0(temp_dir, "/", id), recursive = TRUE, showWarnings = FALSE) + file.copy(example_config, config_path) + file.copy(example_script, script_path) +}) +``` + +::: {.panel-tabset} +```{r show-placeholder, echo=FALSE, output="asis"} +pwalk(langs, function(id, label, config_path, script_path, ...) { + qrt( + "## {% label %} + | + |```{embed, lang='{%id%}'} + |{%script_path%} + |``` + |") +}) +``` +::: + +A "Viash placeholder" code block is the section between the `VIASH START` and `VIASH END` comments. + +## What happens at runtime +By passing arguments to the component, Viash will add your parameter values to your script by replacing the Viash placeholder code block. If no such code block exists yet, the parameters are inserted at the top of the file. + +The resulting code block will contain two maps (or dictionaries): `par` and `meta`. The `par` map contains the parameter values specified by the user, and `meta` contains additional information on the current runtime environment. Note that for Bash scripts, the `par` and `meta` maps are flattened into separate environment variables. + +## Previewing the `par` and `meta` objects +To get insight into how `par` and `meta` are defined, you can run [`viash config inject`](/reference/cli/config_inject.qmd) to replace the current parameter placeholder with an auto-generated parameter placeholder. + +::: {.callout-warning} +This will change the contents of your script! +::: + +::: {.panel-tabset} +```{r config-inject, echo=FALSE, output="asis"} +pwalk(langs, function(id, label, config_path, script_path, ...) { + qrt( + "## {% label %} + | + |Running `viash config inject` effectively changes the contents of the script. + | + |```{bash config-inject} + |viash config inject {%basename(config_path)%} + |``` + | + |The updated `{%basename(script_path)%}` now contains the following code: + | + |```{embed, lang='{%id%}'} + |{%script_path%} + |``` + |", .dir = paste0(temp_dir, "/", id)) +}) +``` +::: + +## Runtime parameters in `par` + +The `par` object (or `par_` environment variables in Bash) will contain argument values passed at runtime. For example, passing `--input foo.txt` will result in a `par["input"]` being equal to `"foo.txt"`. + +:::{.callout-tip} +Try adding more [arguments]({{< var reference.arguments >}}) with different file types to see what effect this has on the resulting placeholder. +::: + +## Meta variables in `meta` + +Meta-variables offer information on the runtime environment which you can use from within your script. + +* `cpus` (integer): The maximum number of (logical) cpus a component is allowed to use. By default, this value will be undefined. + +* `config` (string): Path to the processed Viash config YAML. This file is usually called `.config.vsh.yaml` and resides next to the wrapped executable (see below). This YAML file is useful for doing some runtime introspection of the component for writing generic unit tests. + +* `executable` (string): The executable being used at runtime; that is, the wrapped script. This variable is used in unit tests. + +* `functionality_name` (string): The name of the component, useful for logging. + +* `memory_*` (long): The maximum amount of memory a component is allowed to allocate. The following denominations are provided: `memory_b`, `memory_kb`, `memory_mb`, `memory_gb`, `memory_tb`, `memory_pb`. By default, this value will be undefined. + +* `resources_dir` (string): Path to where the resources are stored. + +* `temp_dir` (string): A temporary directory in which your script is allowed to create new temporary files / directories. By default, this will be set to the `VIASH_TEMP` environment variable. When the `VIASH_TEMP` variable is undefined, POSIX `TMPDIR` or `/tmp` is used instead. + + +### `cpus` (integer) + + +This field specifies the maximum number of (logical) cpus a component is allowed to use. This is useful when parallellizing your component in such a way that integrates very nicely with pipeline frameworks such as Nextflow. Below is an example usage of the `cpus` meta-variable. + +::: {.panel-tabset} +## Bash +```bash +#!/bin/bash + +## VIASH START +par_input="path/to/file.txt" +par_output="output.txt" +meta_cpus=10 +## VIASH END + +# Pass number of cores to the popular_software_tool. Set the default to 1. +./popular_software_tool --ncores ${meta_cpus:-1} +``` +## C\# + +No example available yet. + +## JavaScript + +No example available yet. + +## Python + +```python +from multiprocessing import Pool + +## VIASH START +par = {} +meta = {"cpus": 1} +## VIASH END + +def my_fun(x): + return x + "!" +my_data = ["hello", "world"] + +with Pool(processes=meta.get("cpus", 1)) as pool: + out = pool.map(my_fun, my_data) +``` + +## R + +```r +library(furrr) + +## VIASH START +par <- list() +meta <- list( + cpus = 1L +) +## VIASH END + +if (is.null(meta$cpus)) meta$cpus <- 1 +plan(multisession, workers = meta$cpus) + +my_data <- c("hello", "world") +out = future_map( + my_data, + function(x) { + paste0(x, "!") + } +) +``` + +## Scala + +```scala +import scala.collection.parallel._ +import java.util.concurrent.ForkJoinPool + +// VIASH START +// ... +// VIASH END + +val pc = mutable.ParArray(1, 2, 3) +val numCores = meta.cores.getOrElse(1) +pc.tasksupport = new ForkJoinTaskSupport(new ForkJoinPool(numCores)) +pc map { _ + 1 } +``` +::: + +You can set the number of cores in your component using any of the following approaches: + +```bash +# as a parameter of viash run +viash run config.vsh.yaml --cpus 10 -- + +# as a parameter of viash test +viash test config.vsh.yaml --cpus 10 + +# or as a parameter of the executable +viash build config.vsh.yaml -o output +output/my_executable ---cpus 10 +# ↑ notice the triple dash +``` + +### `config` (string) + +Path to the processed Viash config YAML. +This file is usually called `.config.vsh.yaml` and resides next to the wrapped executable (see below). +This YAML file is useful for doing some runtime introspection of the component for writing generic unit tests. + +### `executable` (string) + +The executable being used at runtime; that is, the wrapped script. This variable is used in unit tests. + +```bash +#!/usr/bin/env bash +set -x + +"$meta_executable" --input input.txt > output.txt + +[[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 +cat output.txt +grep -q 'expected output' output.txt + +echo Done +``` + +### `functionality_name` (string) + +The name of the component, useful for logging. + +### `memory_*` (long) + +The maximum amount of memory a component is allowed to allocate. +The following denominations are provided: `memory_b`, `memory_kb`, `memory_mb`, `memory_gb`, `memory_tb`, `memory_pb`. +By default, this value will be undefined. + +You can set the amount of memory in your component using any of the following approaches: + +```bash +# as a parameter of viash run +viash run config.vsh.yaml --memory 2GB -- + +# as a parameter of viash test +viash test config.vsh.yaml --memory 2GB + +# or as a parameter of the executable +viash build config.vsh.yaml -o output +output/my_executable ---memory 2GB +# ↑ notice the triple dash +``` + +### `resources_dir` (string) + +This field specifies the absolute path to where the resources are stored. +During the build phase resources are copied or fetched into this directory so they are ready to be read during execution of the script or test scripts. + +### `temp_dir` (string) + +A temporary directory in which your script is allowed to create new temporary files / directories. +By default, this will be set to the `VIASH_TEMP` environment variable. +When the `VIASH_TEMP` variable is undefined, the POSIX `TMPDIR` and other common misspellings will be checked and ultimately `/tmp` is used as fallback. diff --git a/project/plugins.sbt b/project/plugins.sbt index 07e85f1b1..fe6752eb8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.5") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.9.0") diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh b/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh index 7da538090..d47d74983 100644 --- a/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh +++ b/src/main/resources/io/viash/helpers/bashutils/ViashAutodetectMount.sh @@ -27,6 +27,7 @@ function ViashAutodetectMountArg { base_name=`basename "$abs_path"` fi mount_target="/viash_automount$mount_source" + ViashDebug "ViashAutodetectMountArg $1 -> $mount_source -> $mount_target" echo "--volume=\"$mount_source:$mount_target\"" } function ViashStripAutomount { diff --git a/src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh b/src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh index a8c34232a..62cdd98d7 100644 --- a/src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh +++ b/src/main/resources/io/viash/helpers/bashutils/ViashLogging.sh @@ -20,7 +20,7 @@ function ViashLog { local display_tag="$2" shift 2 if [ $VIASH_VERBOSITY -ge $required_level ]; then - echo "[$display_tag]" "$@" + >&2 echo "[$display_tag]" "$@" fi } diff --git a/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf b/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf index 60006bca3..2d3860033 100644 --- a/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf +++ b/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf @@ -423,23 +423,24 @@ def processProcessArgs(Map args) { def processArgs = thisDefaultProcessArgs + args // check whether 'key' exists - assert processArgs.containsKey("key") + assert processArgs.containsKey("key") : "Error in module '${thisConfig.functionality.name}': key is a required argument" // if 'key' is a closure, apply it to the original key if (processArgs["key"] instanceof Closure) { processArgs["key"] = processArgs["key"](thisConfig.functionality.name) } - assert processArgs["key"] instanceof CharSequence - assert processArgs["key"] ==~ /^[a-zA-Z_][a-zA-Z0-9_]*$/ + def key = processArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" // check whether directives exists and apply defaults - assert processArgs.containsKey("directives") - assert processArgs["directives"] instanceof Map + assert processArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert processArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${processArgs['directives'].getClass()}" processArgs["directives"] = processDirectives(thisDefaultProcessArgs.directives + processArgs["directives"]) // check whether directives exists and apply defaults - assert processArgs.containsKey("auto") - assert processArgs["auto"] instanceof Map + assert processArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert processArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${processArgs['auto'].getClass()}" processArgs["auto"] = processAuto(thisDefaultProcessArgs.auto + processArgs["auto"]) // auto define publish, if so desired @@ -491,12 +492,88 @@ def processProcessArgs(Map args) { processArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) } - for (nam in [ "map", "mapId", "mapData", "mapPassthrough", "filter" ]) { + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter"]) { if (processArgs.containsKey(nam) && processArgs[nam]) { - assert processArgs[nam] instanceof Closure : "Expected process argument '$nam' to be null or a Closure. Found: class ${processArgs[nam].getClass()}" + assert processArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${processArgs[nam].getClass()}" + } + } + + // check fromState + assert processArgs.containsKey("fromState") : "Error in module '$key': fromState is a required argument" + def fromState = processArgs["fromState"] + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState) { + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key': the state is not a Map" + def data = fromStateMap.collectEntries{newkey, origkey -> + // check whether all values of fromState are in state + assert state.containsKey(origkey) : "Error in module '$key': fromState key '$origkey' not found in current state" + [newkey, state[origkey]] + } + data + } + } + + processArgs["fromState"] = fromState + } + + // check toState + def toState = processArgs["toState"] + + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key': the output is not a Map" + assert state instanceof Map : "Error in module '$key': the state is not a Map" + def extraEntries = toStateMap.collectEntries{newkey, origkey -> + // check whether all values of toState are in output + assert output.containsKey(origkey) : "Error in module '$key': toState key '$origkey' not found in current output" + [newkey, output[origkey]] + } + state + extraEntries } } + processArgs["toState"] = toState + // return output return processArgs } @@ -819,6 +896,7 @@ def workflowFactory(Map args) { } tuple } + if (processArgs.filter) { mid2_ = mid1_ | filter{processArgs.filter(it)} @@ -826,17 +904,21 @@ def workflowFactory(Map args) { mid2_ = mid1_ } - output_passthrough = mid2_ - | map { tuple -> - [tuple[0]] + tuple.drop(2) - } + if (processArgs.fromState) { + mid3_ = mid2_ + | map{ + def new_data = processArgs["fromState"](it.take(2)) + [it[0], new_data] + } + } else { + mid3_ = mid2_ + } - output_process = mid2_ + out0_ = mid3_ | debug(processArgs, "processed") | map { tuple -> def id = tuple[0] def data = tuple[1] - def passthrough = tuple.drop(2) // fetch default params from functionality def defaultArgs = thisConfig.functionality.allArguments @@ -957,11 +1039,19 @@ def workflowFactory(Map args) { [ output[0], outputFiles ] } - output_ = output_process.join(output_passthrough, failOnDuplicate: true) + // join the output [id, output] with the previous state [id, state, ...] + out1_ = out0_.join(mid2_, failOnDuplicate: true) + // input tuple format: [id, output, prev_state, ...] + // output tuple format: [id, new_state, ...] + | map{ + def new_state = processArgs["toState"](it) + [it[0], new_state] + it.drop(3) + } | debug(processArgs, "output") + emit: - output_ + out1_ } def wf = workflowInstance.cloneWithName(workflowKey) diff --git a/src/main/scala/io/viash/Main.scala b/src/main/scala/io/viash/Main.scala index 651ca62ff..3f04a1509 100644 --- a/src/main/scala/io/viash/Main.scala +++ b/src/main/scala/io/viash/Main.scala @@ -17,34 +17,27 @@ package io.viash -import java.io.File -import java.io.FileNotFoundException -import java.nio.file.NoSuchFileException -import java.nio.file.Paths -import java.nio.file.FileSystemNotFoundException +import java.io.{File, FileNotFoundException} +import java.nio.file.{Path, Paths, Files, NoSuchFileException, FileSystemNotFoundException} +import java.net.URI import sys.process.{Process, ProcessLogger} import config.Config -import helpers.IO +import helpers.{IO, Exec, SysEnv, Logger, Logging} import helpers.Scala._ -import cli.{CLIConf, ViashCommand, ViashNs, ViashNsBuild} -import io.viash.helpers.MissingResourceFileException -import io.viash.helpers.status._ -import io.viash.platforms.Platform -import io.viash.project.ViashProject -import io.viash.cli.DocumentedSubcommand -import java.nio.file.Path -import io.viash.helpers.Exec -import java.nio.file.Files -import java.net.URI +import helpers.status._ +import platforms.Platform +import project.ViashProject +import cli.{CLIConf, ViashCommand, DocumentedSubcommand, ViashNs, ViashNsBuild, ViashLogger} +import exceptions._ +import org.rogach.scallop._ +import io.viash.helpers.LoggerLevel -object Main { +object Main extends Logging { private val pkg = getClass.getPackage val name: String = if (pkg.getImplementationTitle != null) pkg.getImplementationTitle else "viash" val version: String = if (pkg.getImplementationVersion != null) pkg.getImplementationVersion else "test" - val viashHome = Paths.get(sys.env.getOrElse("VIASH_HOME", sys.env("HOME") + "/.viash")) - /** * Viash main * @@ -65,14 +58,23 @@ object Main { System.exit(exitCode) } catch { case e @ ( _: FileNotFoundException | _: MissingResourceFileException ) => - Console.err.println(s"viash: ${e.getMessage()}") + info(s"viash: ${e.getMessage()}") System.exit(1) case e: NoSuchFileException => // This exception only returns the file/path that can't be found. Add a bit more feedback to the user. - Console.err.println(s"viash: ${e.getMessage()} (No such file or directory)") + info(s"viash: ${e.getMessage()} (No such file or directory)") + System.exit(1) + case e: AbstractConfigException => + info(s"viash: Error parsing, ${e.innerMessage} in file '${e.uri}'.") + info(s"Details:\n${e.getMessage()}") + System.exit(1) + case e: MalformedInputException => + info(s"viash: ${e.getMessage()}") System.exit(1) + case ee: ExitException => + System.exit(ee.code) case e: Exception => - Console.err.println( + info( s"""Unexpected error occurred! If you think this is a bug, please post |create an issue at https://github.com/viash-io/viash/issues containing |a reproducible example and the stack trace below. @@ -127,7 +129,7 @@ object Main { * @return An exit code */ def mainVersioned(args: Array[String], workingDir: Option[Path] = None, version: String): Int = { - val path = viashHome.resolve("releases").resolve(version).resolve("viash") + val path = Paths.get(SysEnv.viashHome).resolve("releases").resolve(version).resolve("viash") if (!Files.exists(path)) { // todo: be able to use 0.7.x notation to get the latest 0.7 version? @@ -145,7 +147,7 @@ object Main { Array(path.toString) ++ args, cwd = workingDir.map(_.toFile), extraEnv = List("VIASH_VERSION" -> "-"): _* - ).!(ProcessLogger(Console.out.println, Console.err.println)) + ).!(ProcessLogger(s => infoOut(s), s => info(s))) } /** @@ -173,6 +175,23 @@ object Main { // parse arguments val cli = new CLIConf(viashArgs.toIndexedSeq) + + // Set Logger paramters + cli.subcommands.last match { + case x: ViashLogger => + if (x.colorize.isDefined) { + val colorize = x.colorize() match { + case "auto" => None + case "true" => Some(true) + case "false" => Some(false) + } + Logger.UseColorOverride.value = colorize + } + if (x.loglevel.isDefined) { + Logger.UseLevelOverride.value = LoggerLevel.fromString(x.loglevel()) + } + case _ => + } // see if there are project overrides passed to the viash command val projSrc = cli.subcommands.last match { @@ -223,6 +242,7 @@ object Main { config, platform = platform.get, keepFiles = cli.test.keep.toOption.map(_.toBoolean), + setupStrategy = cli.test.setup.toOption, cpus = cli.test.cpus.toOption, memory = cli.test.memory.toOption ) @@ -250,7 +270,8 @@ object Main { tsv = cli.namespace.test.tsv.toOption, append = cli.namespace.test.append(), cpus = cli.namespace.test.cpus.toOption, - memory = cli.namespace.test.memory.toOption + memory = cli.namespace.test.memory.toOption, + setup = cli.namespace.test.setup.toOption, ) val errors = testResults.flatMap(_.toOption).count(_.isError) if (errors > 0) 1 else 0 @@ -306,18 +327,41 @@ object Main { 0 case List(cli.export, cli.export.cli_schema) => val output = cli.export.cli_schema.output.toOption.map(Paths.get(_)) - ViashExport.exportCLISchema(output) + ViashExport.exportCLISchema( + output, + format = cli.export.cli_schema.format() + ) + 0 + case List(cli.export, cli.export.cli_autocomplete) => + val output = cli.export.cli_autocomplete.output.toOption.map(Paths.get(_)) + ViashExport.exportAutocomplete( + output, + format = cli.export.cli_autocomplete.format() + ) 0 case List(cli.export, cli.export.config_schema) => val output = cli.export.config_schema.output.toOption.map(Paths.get(_)) - ViashExport.exportConfigSchema(output) + ViashExport.exportConfigSchema( + output, + format = cli.export.config_schema.format() + ) + 0 + case List(cli.export, cli.export.json_schema) => + val output = cli.export.json_schema.output.toOption.map(Paths.get(_)) + ViashExport.exportJsonSchema( + output, + format = cli.export.json_schema.format() + ) 0 case List(cli.export, cli.export.resource) => val output = cli.export.resource.output.toOption.map(Paths.get(_)) - ViashExport.exportResource(cli.export.resource.path.toOption.get, output) + ViashExport.exportResource( + cli.export.resource.path.toOption.get, + output + ) 0 case _ => - Console.err.println("No subcommand was specified. See `viash --help` for more information.") + info("No subcommand was specified. See `viash --help` for more information.") 1 } } @@ -452,9 +496,7 @@ object Main { */ def detectVersion(workingDir: Option[Path]): Option[String] = { // if VIASH_VERSION is defined, use that - if (sys.env.get("VIASH_VERSION").isDefined) { - sys.env.get("VIASH_VERSION") - } else { + SysEnv.viashVersion orElse { // else look for project file in working dir // and try to read as json workingDir diff --git a/src/main/scala/io/viash/ViashBuild.scala b/src/main/scala/io/viash/ViashBuild.scala index 0050addf6..28a7409e2 100644 --- a/src/main/scala/io/viash/ViashBuild.scala +++ b/src/main/scala/io/viash/ViashBuild.scala @@ -23,9 +23,9 @@ import io.viash.helpers.status._ import config._ import platforms.Platform -import helpers.IO +import helpers.{IO, Logging} -object ViashBuild { +object ViashBuild extends Logging { def apply( config: Config, platform: Platform, @@ -52,7 +52,7 @@ object ViashBuild { val setupResult = if (setup.isDefined && exec_path.isDefined && platform.hasSetup) { val cmd = Array(exec_path.get, "---setup", setup.get) - val res = Process(cmd).!(ProcessLogger(println, println)) + val res = Process(cmd).!(ProcessLogger(s => infoOut(s), s => infoOut(s))) res } else 0 @@ -61,7 +61,7 @@ object ViashBuild { val pushResult = if (push && exec_path.isDefined && platform.hasSetup) { val cmd = Array(exec_path.get, "---setup push") - val _ = Process(cmd).!(ProcessLogger(println, println)) + val _ = Process(cmd).!(ProcessLogger(s => infoOut(s), s => infoOut(s))) } else 0 diff --git a/src/main/scala/io/viash/ViashConfig.scala b/src/main/scala/io/viash/ViashConfig.scala index e94162c4f..c6351e0d6 100644 --- a/src/main/scala/io/viash/ViashConfig.scala +++ b/src/main/scala/io/viash/ViashConfig.scala @@ -17,34 +17,19 @@ package io.viash -import io.viash.config.Config -import io.viash.helpers.IO - import java.nio.file.{Files, Paths} +import scala.sys.process.Process + import io.circe.syntax.EncoderOps -import io.circe.{Json, Printer => JsonPrinter} -import io.circe.yaml.{Printer => YamlPrinter} -import scala.sys.process.Process +import io.viash.config.Config +import io.viash.helpers.{IO, Logging} +import io.viash.helpers.circe._ import io.viash.platforms.DebugPlatform import io.viash.config.ConfigMeta +import io.viash.exceptions.ExitException -object ViashConfig { - private val yamlPrinter = YamlPrinter( - preserveOrder = true, - mappingStyle = YamlPrinter.FlowStyle.Block, - splitLines = true, - stringStyle = YamlPrinter.StringStyle.DoubleQuoted - ) - private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) - - private def printJson(json: Json, format: String): Unit = { - val str = format match { - case "yaml" => yamlPrinter.pretty(json) - case "json" => jsonPrinter.print(json) - } - println(str) - } +object ViashConfig extends Logging{ def view(config: Config, format: String, parseArgumentGroups: Boolean): Unit = { val conf0 = @@ -59,7 +44,7 @@ object ViashConfig { config } val json = ConfigMeta.configToCleanJson(conf0) - printJson(json, format) + infoOut(json.toFormattedString(format)) } def viewMany(configs: List[Config], format: String, parseArgumentGroups: Boolean): Unit = { @@ -76,7 +61,7 @@ object ViashConfig { } } val jsons = confs0.map(c => ConfigMeta.configToCleanJson(c)) - printJson(jsons.asJson, format) + infoOut(jsons.asJson.toFormattedString(format)) } def inject(config: Config): Unit = { @@ -84,25 +69,25 @@ object ViashConfig { // check if config has a main script if (fun.mainScript.isEmpty) { - println("Could not find a main script in the Viash config.") - System.exit(1) + infoOut("Could not find a main script in the Viash config.") + throw new ExitException(1) } // check if we can read code if (fun.mainScript.get.read.isEmpty) { - println("Could not read main script in the Viash config.") - System.exit(1) + infoOut("Could not read main script in the Viash config.") + throw new ExitException(1) } // check if main script has a path if (fun.mainScript.get.uri.isEmpty) { - println("Main script should have a path.") - System.exit(1) + infoOut("Main script should have a path.") + throw new ExitException(1) } val uri = fun.mainScript.get.uri.get // check if main script is a local file if (uri.getScheme != "file") { - println("Config inject only works for local Viash configs.") - System.exit(1) + infoOut("Config inject only works for local Viash configs.") + throw new ExitException(1) } val path = Paths.get(uri.getPath()) diff --git a/src/main/scala/io/viash/ViashExport.scala b/src/main/scala/io/viash/ViashExport.scala index 0b46c715b..2cdbb9663 100644 --- a/src/main/scala/io/viash/ViashExport.scala +++ b/src/main/scala/io/viash/ViashExport.scala @@ -20,32 +20,58 @@ package io.viash import helpers._ import cli._ import io.circe.{Printer => JsonPrinter} +import io.circe.yaml.{Printer => YamlPrinter} import io.circe.syntax.EncoderOps import io.viash.helpers.circe._ import java.nio.file.{Path, Paths, Files} -import io.viash.schemas.CollectedSchemas +import io.viash.schemas._ +import io.circe.Json -object ViashExport { - private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) +object ViashExport extends Logging { + def exportCLISchema(output: Option[Path], format: String): Unit = { + val cli = new CLIConf(Nil) + val data = cli.getRegisteredCommands().asJson + val str = data.toFormattedString(format) + if (output.isDefined) { + Files.write(output.get, str.getBytes()) + } else { + infoOut(str) + } + } - def exportCLISchema(output: Option[Path]): Unit = { + def exportAutocomplete(output: Option[Path], format: String): Unit = { val cli = new CLIConf(Nil) - val data = cli.getRegisteredCommands - val str = jsonPrinter.print(data.asJson) + val str = + format match { + case "bash" => AutoCompleteBash.generate(cli) + case "zsh" => AutoCompleteZsh.generate(cli) + case _ => throw new IllegalArgumentException("'format' must be either 'bash' or 'zsh'.") + } if (output.isDefined) { Files.write(output.get, str.getBytes()) } else { - println(str) + infoOut(str) } } - def exportConfigSchema(output: Option[Path]): Unit = { + def exportConfigSchema(output: Option[Path], format: String): Unit = { val data = CollectedSchemas.getJson - val str = jsonPrinter.print(data.asJson) + val str = data.toFormattedString(format) + if (output.isDefined) { + Files.write(output.get, str.getBytes()) + } else { + infoOut(str) + } + } + + + def exportJsonSchema(output: Option[Path], format: String): Unit = { + val data = JsonSchema.getJsonSchema + val str = data.toFormattedString(format) if (output.isDefined) { Files.write(output.get, str.getBytes()) } else { - println(str) + infoOut(str) } } @@ -55,7 +81,7 @@ object ViashExport { if (output.isDefined) { Files.write(output.get, str.getBytes()) } else { - println(str) + infoOut(str) } } } diff --git a/src/main/scala/io/viash/ViashNamespace.scala b/src/main/scala/io/viash/ViashNamespace.scala index 0fc42b455..4a975b070 100644 --- a/src/main/scala/io/viash/ViashNamespace.scala +++ b/src/main/scala/io/viash/ViashNamespace.scala @@ -20,8 +20,8 @@ package io.viash import java.nio.file.{Paths, Files, StandardOpenOption} import io.viash.ViashTest.{ManyTestOutput, TestOutput} import config.Config -import helpers.IO -import io.viash.helpers.MissingResourceFileException +import helpers.{IO, Logging} +import io.viash.exceptions.MissingResourceFileException import io.viash.helpers.status._ import java.nio.file.Path import io.viash.helpers.NsExecData._ @@ -30,8 +30,10 @@ import sys.process._ import java.io.{ByteArrayOutputStream, File, PrintWriter} import io.viash.platforms.Platform import scala.collection.parallel.CollectionConverters._ +import io.viash.helpers.LoggerOutput +import io.viash.helpers.LoggerLevel -object ViashNamespace { +object ViashNamespace extends Logging { case class MaybeParList[T]( list: List[T], @@ -87,7 +89,7 @@ object ViashNamespace { targetOutputPath(target, platformId, ns, funName) } val nsStr = ns.map(" (" + _ + ")").getOrElse("") - println(s"Exporting $funName$nsStr =$platformId=> $out") + infoOut(s"Exporting $funName$nsStr =$platformId=> $out") val status = ViashBuild( config = conf, platform = platform, @@ -106,6 +108,7 @@ object ViashNamespace { def test( configs: List[Either[(Config, Option[Platform]), Status]], parallel: Boolean = false, + setup: Option[String] = None, keepFiles: Option[Boolean] = None, tsv: Option[String] = None, append: Boolean = false, @@ -133,7 +136,7 @@ object ViashNamespace { val parentTempPath = IO.makeTemp("viash_ns_test") if (keepFiles.getOrElse(true)) { - Console.err.printf("The working directory for the namespace tests is %s\n", parentTempPath.toString()) + info(s"The working directory for the namespace tests is ${parentTempPath.toString()}") } try { @@ -151,18 +154,15 @@ object ViashNamespace { ).mkString("\t") + sys.props("line.separator")) writer.flush() } - printf( - s"%s%20s %20s %20s %20s %9s %8s %20s%s\n", - "", - "namespace", - "functionality", - "platform", - "test_name", - "exit_code", - "duration", - "result", - Console.RESET - ) + infoOut("%20s %20s %20s %20s %9s %8s %20s". + format("namespace", + "functionality", + "platform", + "test_name", + "exit_code", + "duration", + "result" + )) val results = configs2.map { x => x match { @@ -175,7 +175,7 @@ object ViashNamespace { val platName = platform.id // print start message - printf(s"%s%20s %20s %20s %20s %9s %8s %20s%s\n", "", namespace, funName, platName, "start", "", "", "", Console.RESET) + infoOut("%20s %20s %20s %20s %9s %8s %20s".format(namespace, funName, platName, "start", "", "", "")) // run tests // TODO: it would actually be great if this component could subscribe to testresults messages @@ -184,6 +184,7 @@ object ViashNamespace { ViashTest( config = conf, platform = platform, + setupStrategy = setup, keepFiles = keepFiles, quiet = true, parentTempPath = Some(parentTempPath), @@ -192,7 +193,7 @@ object ViashNamespace { ) } catch { case e: MissingResourceFileException => - Console.err.println(s"${Console.YELLOW}viash ns: ${e.getMessage}${Console.RESET}") + warn(s"viash ns: ${e.getMessage}") ManyTestOutput(None, List()) } @@ -219,11 +220,11 @@ object ViashNamespace { } // print message - printf(s"%s%20s %20s %20s %20s %9s %8s %20s%s\n", col, namespace, funName, platName, test.name, test.exitValue, test.duration, msg, Console.RESET) + log(LoggerOutput.StdOut, LoggerLevel.Info, col, "%20s %20s %20s %20s %9s %8s %20s".format(namespace, funName, platName, test.name, test.exitValue, test.duration, msg)) if (test.exitValue != 0) { - Console.err.println(test.output) - Console.err.println(ViashTest.consoleLine) + info(test.output) + info(ViashTest.consoleLine) } // write to tsv @@ -260,7 +261,7 @@ object ViashNamespace { results } catch { case e: Exception => - println(e.getMessage()) + infoOut(e.getMessage()) Nil } finally { tsvWriter.foreach(_.close()) @@ -307,7 +308,7 @@ object ViashNamespace { // check whether is empty if (configData.isEmpty) { - Console.err.println("No config files found to work with.") + info("No config files found to work with.") return } @@ -315,7 +316,7 @@ object ViashNamespace { // Slashes for ';' or '+' are not needed here, but let's allow it anyway val matchChecker = """([^{}]*\{[\w-]*\})*[^{}]*(\\?[;+])?$""" if (!command.matches(matchChecker)) { - Console.err.println("Invalid command syntax.") + info("Invalid command syntax.") return } @@ -323,7 +324,7 @@ object ViashNamespace { val fields = """\{[^\{\}]*\}""".r.findAllIn(command).map(_.replaceAll("^.|.$", "")).toList val unfoundFields = fields.filter(configData.head.getField(_).isEmpty) if (!unfoundFields.isEmpty) { - Console.err.println(s"Not all substitution fields are supported fields: ${unfoundFields.mkString(" ")}.") + info(s"Not all substitution fields are supported fields: ${unfoundFields.mkString(" ")}.") return } @@ -348,16 +349,16 @@ object ViashNamespace { errorOrCmd match { case Left(error) => - Console.err.println(s"+ $command") - Console.err.println(s" Error: $error") + info(s"+ $command") + info(s" Error: $error") case Right(cmd) if dryrun => - Console.err.println(s"+ $cmd") + info(s"+ $cmd") case Right(cmd) => - Console.err.println(s"+ $cmd") + info(s"+ $cmd") val (exitcode, output) = runExecCommand(cmd) - Console.err.println(s" Exit code: $exitcode\n") - Console.err.println(s" Output:") - Console.out.println(output) + info(s" Exit code: $exitcode\n") + info(s" Output:") + infoOut(output) } } } @@ -378,7 +379,7 @@ object ViashNamespace { (exitValue, stream.toString) } catch { case e: Throwable => - Console.err.println(s" Exception: $e") + info(s" Exception: $e") (-1, e.getMessage()) } finally { printwriter.close() @@ -399,20 +400,29 @@ object ViashNamespace { (ParseError, "configs encountered parse errors"), (Disabled, "configs were disabled"), (BuildError, "configs built failed"), + (SetupError, "setups failed"), + (PushError, "pushes failed"), (TestError, "tests failed"), (TestMissing, "tests missing"), (Success, s"configs $successAction successfully")) if (successes != statuses.length) { - Console.err.println(s"${Console.YELLOW}Not all configs $successAction successfully${Console.RESET}") + val disabledStatusesCount = statuses.count(_ == Disabled) + val nonDisabledStatuses = statuses.filter(_ != Disabled) + val indentSize = nonDisabledStatuses.length.toString().size + + warn(s"Not all configs $successAction successfully") + if (disabledStatusesCount > 0) + warn(s" $disabledStatusesCount configs were disabled") + for ((status, message) <- messages) { - val count = statuses.count(_ == status) + val count = nonDisabledStatuses.count(_ == status) if (count > 0) - Console.err.println(s" ${status.color}$count/${statuses.length} ${message}${Console.RESET}") + log(LoggerOutput.StdErr, LoggerLevel.Info, status.color, s" ${String.format(s"%${indentSize}s",count)}/${nonDisabledStatuses.length} ${message}") } } else { - Console.err.println(s"${Console.GREEN}All ${successes} configs $successAction successfully${Console.RESET}") + log(LoggerOutput.StdErr, LoggerLevel.Info, Console.GREEN, s"All ${successes} configs $successAction successfully") } } } diff --git a/src/main/scala/io/viash/ViashRun.scala b/src/main/scala/io/viash/ViashRun.scala index abaa3a351..2f11c548e 100644 --- a/src/main/scala/io/viash/ViashRun.scala +++ b/src/main/scala/io/viash/ViashRun.scala @@ -22,12 +22,12 @@ import java.nio.file.Paths import io.viash.config._ import io.viash.functionality.arguments.{FileArgument, Output} import io.viash.platforms.Platform -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logging} import io.viash.helpers.data_structures._ import scala.sys.process.{Process, ProcessLogger} -object ViashRun { +object ViashRun extends Logging { def apply( config: Config, platform: Platform, @@ -55,7 +55,7 @@ object ViashRun { Array(cpus.map("---cpus=" + _), memory.map("---memory="+_)).flatMap(a => a) // execute command, print everything to console - code = Process(cmd).!(ProcessLogger(println, println)) + code = Process(cmd).!(ProcessLogger(s => infoOut(s), s => infoOut(s))) // System.exit(code) code } finally { @@ -63,7 +63,7 @@ object ViashRun { if (!keepFiles.getOrElse(code != 0)) { IO.deleteRecursively(dir) } else { - println(s"Files and logs are stored at '$dir'") + infoOut(s"Files and logs are stored at '$dir'") } } } diff --git a/src/main/scala/io/viash/ViashTest.scala b/src/main/scala/io/viash/ViashTest.scala index 107b640dd..d8491a83b 100644 --- a/src/main/scala/io/viash/ViashTest.scala +++ b/src/main/scala/io/viash/ViashTest.scala @@ -29,13 +29,13 @@ import functionality.Functionality import functionality.arguments.{FileArgument, Output} import functionality.resources.{BashScript, Script} import platforms.NativePlatform -import helpers.IO +import helpers.{IO, Logging, LoggerOutput, LoggerLevel} import io.viash.helpers.data_structures._ -import io.viash.helpers.MissingResourceFileException +import io.viash.exceptions.MissingResourceFileException import io.viash.platforms.Platform import io.viash.config.ConfigMeta -object ViashTest { +object ViashTest extends Logging { case class TestOutput(name: String, exitValue: Int, output: String, logFile: String, duration: Long) case class ManyTestOutput(setup: Option[TestOutput], tests: List[TestOutput]) @@ -61,7 +61,7 @@ object ViashTest { platform: Platform, keepFiles: Option[Boolean] = None, quiet: Boolean = false, - setupStrategy: String = "cachedbuild", + setupStrategy: Option[String] = None, tempVersion: Option[String] = Some("test"), verbosityLevel: Int = 6, parentTempPath: Option[Path] = None, @@ -70,7 +70,7 @@ object ViashTest { ): ManyTestOutput = { // create temporary directory val dir = IO.makeTemp("viash_test_" + config.functionality.name, parentTempPath) - if (!quiet) println(s"Running tests in temporary directory: '$dir'") + if (!quiet) infoOut(s"Running tests in temporary directory: '$dir'") // set version to temporary value val config2 = @@ -90,7 +90,7 @@ object ViashTest { platform = platform, dir = dir, verbose = !quiet, - setupStrategy = setupStrategy, + setupStrategy = setupStrategy.getOrElse("cachedbuild"), verbosityLevel = verbosityLevel, cpus = cpus, memory = memory @@ -109,18 +109,18 @@ object ViashTest { if (!quiet) { if (results.isEmpty && !anyErrors) { - println(s"${Console.RED}WARNING! No tests found!${Console.RESET}") + warnOut(s"WARNING! No tests found!") } else if (anyErrors) { - println(s"${Console.RED}ERROR! $errorMessage${Console.RESET}") + errorOut(s"ERROR! $errorMessage") } else { - println(s"${Console.GREEN}SUCCESS! All $count out of ${results.length} test scripts succeeded!${Console.RESET}") + successOut(s"SUCCESS! All $count out of ${results.length} test scripts succeeded!") } } // keep temp files if user asks or any errors are encountered if (!keepFiles.getOrElse(anyErrors)) { - if (!quiet) println("Cleaning up temporary directory") + if (!quiet) infoOut("Cleaning up temporary directory") IO.deleteRecursively(dir) } // TODO: remove container @@ -182,7 +182,7 @@ object ViashTest { val logger: String => Unit = (s: String) => { - if (verbose) println(s) + if (verbose) infoOut(s) printWriter.println(s) logWriter.append(s + sys.props("line.separator")) } @@ -297,7 +297,7 @@ object ViashTest { val logger: String => Unit = (s: String) => { - if (verbose) println(s) + if (verbose) infoOut(s) printWriter.println(s) logWriter.append(s + sys.props("line.separator")) } @@ -331,7 +331,7 @@ object ViashTest { } } - if (verbose) println(consoleLine) + if (verbose) infoOut(consoleLine) ManyTestOutput(buildResult, testResults) } diff --git a/src/main/scala/io/viash/cli/CLIConf.scala b/src/main/scala/io/viash/cli/CLIConf.scala index 3902d6f18..411b5b78e 100644 --- a/src/main/scala/io/viash/cli/CLIConf.scala +++ b/src/main/scala/io/viash/cli/CLIConf.scala @@ -19,7 +19,8 @@ package io.viash.cli import org.rogach.scallop._ import io.viash.Main - +import io.viash.exceptions.ExitException +import io.viash.helpers.Logging trait ViashCommand { _: DocumentedSubcommand => @@ -134,13 +135,35 @@ trait WithTemporary { ) } -class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { - def getRegisteredCommands = subconfigs.flatMap{ sc => +trait ViashLogger { + _: DocumentedSubcommand => + val colorize = registerChoice( + name = "colorize", + short = None, + descr = "Specify whether the console output should be colorized. If not specified, we attempt to detect this automatically.", + choices = List("true", "false", "auto"), + hidden = true + ) + val loglevel = registerChoice( + name = "loglevel", + short = None, + descr = "Specify the log level in use", + choices = List("error", "warn", "info", "debug", "trace"), + hidden = true + ) +} + +class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Logging { + def getRegisteredCommands(includeHidden: Boolean = false) = subconfigs.flatMap{ sc => sc match { - case ds: DocumentedSubcommand if !ds.hidden => Some(ds.toRegisteredCommand) + case ds: DocumentedSubcommand if !ds.hidden || includeHidden => Some(ds.toRegisteredCommand) case _ => None } } + + exitHandler = (i: Int) => throw new ExitException(i) + stderrPrintln = (s: String) => logger.info(s) + stdoutPrintln = (s: String) => logger.infoOut(s) version(s"${Main.name} ${Main.version} (c) 2020 Data Intuitive") @@ -168,7 +191,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { | |Arguments:""".stripMargin) - val run = new DocumentedSubcommand("run") with ViashCommand with WithTemporary with ViashRunner { + val run = new DocumentedSubcommand("run") with ViashCommand with WithTemporary with ViashRunner with ViashLogger { banner( "viash run", "Executes a viash component from the provided viash config file. viash generates a temporary executable and immediately executes it with the given parameters.", @@ -184,7 +207,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { | viash run config.vsh.yaml""".stripMargin) } - val build = new DocumentedSubcommand("build") with ViashCommand { + val build = new DocumentedSubcommand("build") with ViashCommand with ViashLogger { banner( "viash build", "Build an executable from the provided viash config file.", @@ -210,12 +233,19 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { ) } - val test = new DocumentedSubcommand("test") with ViashCommand with WithTemporary with ViashRunner { + val test = new DocumentedSubcommand("test") with ViashCommand with WithTemporary with ViashRunner with ViashLogger { banner( "viash test", "Test the component using the tests defined in the viash config file.", - "viash test config.vsh.yaml [-p docker] [-k true/false]") - + "viash test config.vsh.yaml [-p docker] [-k true/false] [--setup cachedbuild]") + + val setup = registerOpt[String]( + name = "setup", + short = Some('s'), + default = None, + descr = "Which @[setup strategy](docker_setup_strategy) for creating the container to use [Docker Platform only]." + ) + footer( s""" |The temporary directory can be altered by setting the VIASH_TEMP directory. Example: @@ -224,7 +254,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { } val config = new DocumentedSubcommand("config") { - val view = new DocumentedSubcommand("view") with ViashCommand { + val view = new DocumentedSubcommand("view") with ViashCommand with ViashLogger { banner( "viash config view", "View the config file after parsing.", @@ -243,7 +273,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { descr = "Whether or not to postprocess each component's @[argument groups](argument_groups)." ) } - val inject = new DocumentedSubcommand("inject") with ViashCommand { + val inject = new DocumentedSubcommand("inject") with ViashCommand with ViashLogger { banner( "viash config inject", "Inject a Viash header into the main script of a Viash component.", @@ -259,7 +289,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { val namespace = new DocumentedSubcommand("ns") { - val build = new DocumentedSubcommand("build") with ViashNs with ViashNsBuild { + val build = new DocumentedSubcommand("build") with ViashNs with ViashNsBuild with ViashLogger { banner( "viash ns build", "Build a namespace from many viash config files.", @@ -283,11 +313,17 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { ) } - val test = new DocumentedSubcommand("test") with ViashNs with WithTemporary with ViashRunner { + val test = new DocumentedSubcommand("test") with ViashNs with WithTemporary with ViashRunner with ViashLogger { banner( "viash ns test", "Test a namespace containing many viash config files.", - "viash ns test [-n nmspc] [-s src] [-p docker] [--parallel] [--tsv file.tsv] [--append]") + "viash ns test [-n nmspc] [-s src] [-p docker] [--parallel] [--tsv file.tsv] [--setup cachedbuild] [--append]") + + val setup = registerOpt[String]( + name = "setup", + default = None, + descr = "Which @[setup strategy](docker_setup_strategy) for creating the container to use [Docker Platform only]." + ) val tsv = registerOpt[String]( name = "tsv", @@ -302,7 +338,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { ) } - val list = new DocumentedSubcommand("list") with ViashNs { + val list = new DocumentedSubcommand("list") with ViashNs with ViashLogger { banner( "viash ns list", "List a namespace containing many viash config files.", @@ -322,7 +358,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { ) } - val exec = new DocumentedSubcommand("exec") with ViashNs { + val exec = new DocumentedSubcommand("exec") with ViashNs with ViashLogger { banner( "viash ns exec", """Execute a command for all found Viash components. @@ -385,7 +421,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { val `export` = new DocumentedSubcommand("export") { hidden = true - val resource = new DocumentedSubcommand("resource") { + val resource = new DocumentedSubcommand("resource") with ViashLogger { banner( "viash export resource", """Export an internal resource file""".stripMargin, @@ -405,36 +441,91 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) { ) } - val cli_schema = new DocumentedSubcommand("cli_schema") { + val cli_schema = new DocumentedSubcommand("cli_schema") with ViashLogger { banner( "viash export cli_schema", """Export the schema of the Viash CLI as a JSON""".stripMargin, - """viash export cli_schema [--output file.json]""".stripMargin + """viash export cli_schema [--output file.json] [--format json]""".stripMargin + ) + val output = registerOpt[String]( + name = "output", + default = None, + descr = "Destination path" + ) + val format = registerChoice( + name = "format", + short = Some('f'), + default = Some("yaml"), + choices = List("yaml", "json"), + descr = "Which output format to use." + ) + } + + val cli_autocomplete = new DocumentedSubcommand("cli_autocomplete") with ViashLogger { + banner( + "viash export bash_autocomplete", + """Export the autocomplete script as to be used in Bash or Zsh""".stripMargin, + """viash export bash_autocomplete [--output viash_autocomplete_bash] [--zsh]""".stripMargin ) val output = registerOpt[String]( name = "output", default = None, descr = "Destination path" ) + val format = registerChoice( + name = "format", + short = Some('f'), + default = Some("bash"), + choices = List("bash", "zsh"), + descr = "Which autocomplete format to use." + ) } - val config_schema = new DocumentedSubcommand("config_schema") { + val config_schema = new DocumentedSubcommand("config_schema") with ViashLogger { banner( "viash export config_schema", """Export the schema of a Viash config as a JSON""".stripMargin, - """viash export config_schema [--output file.json]""".stripMargin + """viash export config_schema [--output file.json] [--format json]""".stripMargin ) val output = registerOpt[String]( name = "output", default = None, descr = "Destination path" ) + val format = registerChoice( + name = "format", + short = Some('f'), + default = Some("yaml"), + choices = List("yaml", "json"), + descr = "Which output format to use." + ) + } + + val json_schema = new DocumentedSubcommand("json_schema") with ViashLogger { + banner( + "viash export json_schema", + """Export the json schema to validate a Viash config""".stripMargin, + """viash export json_schema [--output file.json] [--format json]""".stripMargin + ) + val output = registerOpt[String]( + name = "output", + default = None, + descr = "Destination path" + ) + val format = registerChoice( + name = "format", + short = Some('f'), + default = Some("yaml"), + choices = List("yaml", "json"), + descr = "Which output format to use." + ) } addSubcommand(resource) addSubcommand(cli_schema) + addSubcommand(cli_autocomplete) addSubcommand(config_schema) - + addSubcommand(json_schema) requireSubcommand() shortSubcommandsHelp(true) diff --git a/src/main/scala/io/viash/config/Config.scala b/src/main/scala/io/viash/config/Config.scala index ce94130b8..65e55ce1b 100644 --- a/src/main/scala/io/viash/config/Config.scala +++ b/src/main/scala/io/viash/config/Config.scala @@ -20,17 +20,15 @@ package io.viash.config import io.viash.config_mods.ConfigModParser import io.viash.functionality._ import io.viash.platforms._ -import io.viash.helpers.{Git, GitInfo, IO} +import io.viash.helpers.{Git, GitInfo, IO, Logging} import io.viash.helpers.circe._ import io.viash.helpers.status._ +import io.viash.helpers.Yaml import java.net.URI -import io.circe.yaml.parser import io.viash.functionality.resources._ import java.io.File -import io.circe.DecodingFailure -import io.circe.ParsingFailure import io.viash.config_mods.ConfigMods import java.nio.file.Paths @@ -69,7 +67,7 @@ case class Config( | | - @[Native](platform_native) | - @[Docker](platform_docker) - | - @[Nextflow VDSL3](platform_nextflow) + | - @[Nextflow](platform_nextflow) |""".stripMargin) platforms: List[Platform] = Nil, @@ -120,7 +118,7 @@ case class Config( } } -object Config { +object Config extends Logging { def readYAML(config: String): (String, Option[Script]) = { val configUri = IO.uri(config) readYAML(configUri) @@ -195,14 +193,13 @@ object Config { /* STRING */ // read yaml as string val (yamlText, optScript) = readYAML(uri) + + // replace valid yaml definitions for +.inf with "+.inf" so that circe doesn't trip over its toes + val replacedYamlText = Yaml.replaceInfinities(yamlText) /* JSON 0: parsed from string */ // parse yaml into Json - def parsingErrorHandler[C](e: Exception): C = { - Console.err.println(s"${Console.RED}Error parsing '${uri}'.${Console.RESET}\nDetails:") - throw e - } - val json0 = parser.parse(yamlText).fold(parsingErrorHandler, identity) + val json0 = Convert.textToJson(replacedYamlText, uri.toString()) /* JSON 1: after inheritance */ // apply inheritance if need be @@ -214,7 +211,7 @@ object Config { /* CONFIG 0: converted from json */ // convert Json into Config - val conf0 = json2.as[Config].fold(parsingErrorHandler, identity) + val conf0 = Convert.jsonToClass[Config](json2, uri.toString()) /* CONFIG 1: store parent path in resource to be able to access them in the future */ val parentURI = uri.resolve("") @@ -269,10 +266,10 @@ object Config { // print warnings if need be if (conf2.functionality.status == Status.Deprecated) - Console.err.println(s"${Console.YELLOW}Warning: The status of the component '${conf2.functionality.name}' is set to deprecated.${Console.RESET}") + warn(s"Warning: The status of the component '${conf2.functionality.name}' is set to deprecated.") if (conf2.functionality.resources.isEmpty && optScript.isEmpty) - Console.err.println(s"${Console.YELLOW}Warning: no resources specified!${Console.RESET}") + warn(s"Warning: no resources specified!") if (!addOptMainScript) { return conf3 @@ -356,7 +353,7 @@ object Config { } } catch { case _: Exception => - Console.err.println(s"${Console.RED}Reading file '$file' failed${Console.RESET}") + error(s"Reading file '$file' failed") Right(ParseError) } } diff --git a/src/main/scala/io/viash/config/Info.scala b/src/main/scala/io/viash/config/Info.scala index e9ddcb001..6b2d52d58 100644 --- a/src/main/scala/io/viash/config/Info.scala +++ b/src/main/scala/io/viash/config/Info.scala @@ -17,13 +17,24 @@ package io.viash.config +import io.viash.schemas.description + +@description("Meta information fields filled in by Viash during build.") case class Info( + @description("Path to the config used during build.") config: String, + @description("The platform id used during build.") platform: Option[String] = None, + @description("Folder path to the build artifacts.") output: Option[String] = None, + @description("Output folder with main executable path.") executable: Option[String] = None, + @description("The Viash version that was used to build the component.") viash_version: Option[String] = None, + @description("Git commit hash.") git_commit: Option[String] = None, + @description("Git remote name.") git_remote: Option[String] = None, + @description("Git tag.") git_tag: Option[String] = None ) \ No newline at end of file diff --git a/src/main/scala/io/viash/config/package.scala b/src/main/scala/io/viash/config/package.scala index e1d198358..8b6ebab91 100644 --- a/src/main/scala/io/viash/config/package.scala +++ b/src/main/scala/io/viash/config/package.scala @@ -22,14 +22,14 @@ import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} package object config { - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ implicit val customConfig: Configuration = Configuration.default.withDefaults // encoders and decoders for Config implicit val encodeConfig: Encoder.AsObject[Config] = deriveConfiguredEncoder - implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeConfig: Decoder[Config] = deriveConfiguredDecoderFullChecks implicit val encodeInfo: Encoder[Info] = deriveConfiguredEncoder - implicit val decodeInfo: Decoder[Info] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeInfo: Decoder[Info] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/config_mods/Command.scala b/src/main/scala/io/viash/config_mods/Command.scala index 506e64176..59ded2196 100644 --- a/src/main/scala/io/viash/config_mods/Command.scala +++ b/src/main/scala/io/viash/config_mods/Command.scala @@ -28,12 +28,13 @@ abstract class Command { case class Assign(lhs: Path, rhs: Value) extends Command { def apply(json: Json): Json = { val result = rhs.get(json) - lhs.applyCommand(json, { _.set(result) }) + lhs.applyCommand(json, { _.set(result) }, false) } } case class Delete(path: Path) extends Command { def apply(json: Json): Json = { - path.applyCommand(json, { _.delete }) + // delete is destructive to the applied path, so we might need to rewrite the history + path.applyCommand(json, { _.delete }, true) } } case class Append(lhs: Path, rhs: Value) extends Command { @@ -49,7 +50,7 @@ case class Append(lhs: Path, rhs: Value) extends Command { cursor.withFocus { cursorJson => cursorJson.mapArray(_ ++ resultVector) } - }) + }, false) } } case class Prepend(lhs: Path, rhs: Value) extends Command { @@ -65,7 +66,7 @@ case class Prepend(lhs: Path, rhs: Value) extends Command { cursor.withFocus { cursorJson => cursorJson.mapArray(resultVector ++ _) } - }) + }, false) } } diff --git a/src/main/scala/io/viash/config_mods/PathExp.scala b/src/main/scala/io/viash/config_mods/PathExp.scala index 07af9bf9b..cff3cee8a 100644 --- a/src/main/scala/io/viash/config_mods/PathExp.scala +++ b/src/main/scala/io/viash/config_mods/PathExp.scala @@ -20,17 +20,17 @@ package io.viash.config_mods import io.circe.{ACursor, Json} abstract class PathExp { - def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path): ACursor + def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path, rewriteHistory: Boolean): ACursor def get(cursor: ACursor, remaining: Path): Json } case object Root extends PathExp { - def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path): ACursor = { + def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path, rewriteHistory: Boolean): ACursor = { val parent = cursor.up if (parent.failed) { - remaining.applyCommand(cursor, cmd) + remaining.applyCommand(cursor, cmd, rewriteHistory) // todo: go back down again? } else { - applyCommand(parent, cmd, remaining) + applyCommand(parent, cmd, remaining, rewriteHistory) } } def get(cursor: ACursor, remaining: Path): Json = { @@ -43,7 +43,7 @@ case object Root extends PathExp { } } case class Attribute(string: String) extends PathExp { - def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path): ACursor = { + def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path, rewriteHistory: Boolean): ACursor = { val down = cursor.downField(string) val newCursor = if (down.failed) { @@ -55,14 +55,8 @@ case class Attribute(string: String) extends PathExp { } else { down } - val result = remaining.applyCommand(newCursor, cmd) - val tryGoingUp = result.up - if (tryGoingUp.failed) { - // todo: going up should always work, so throw an error if it doesn't? - result - } else { - tryGoingUp - } + val result = remaining.applyCommand(newCursor, cmd, rewriteHistory) + result.up.success.getOrElse(result) } def get(cursor: ACursor, remaining: Path): Json = { val down = cursor.downField(string) @@ -74,30 +68,43 @@ case class Attribute(string: String) extends PathExp { } } case class Filter(condition: Condition) extends PathExp { - def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path): ACursor = { - var elemCursor = cursor.downArray + def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path, rewriteHistory: Boolean): ACursor = { + def recurseApply(cursor: ACursor, cmd: ACursor => ACursor, remaining: Path, rewriteHistory: Boolean): ACursor = { + val isLast = cursor.right.failed + val (modifiedCursor, moveRight) = + if (condition.apply(cursor.focus.get)) { + val modified = remaining.applyCommand(cursor, cmd, rewriteHistory) + val modifiedWithCorrectHistory = + if (rewriteHistory && !isLast) { + // replay history of the original cursor on the modified cursor to make sure we're at the right position + modified.top.get.hcursor.replay(cursor.history) + } else { + modified + } + (modifiedWithCorrectHistory, !remaining.path.isEmpty || !rewriteHistory) + } else { + (cursor, true) + } + + val newCursor = + if (moveRight && !isLast) + modifiedCursor.right + else + modifiedCursor + + if (!isLast) + recurseApply(newCursor, cmd, remaining, rewriteHistory) + else + newCursor + } + + val elemCursor = cursor.downArray if (elemCursor.failed) { // cursor doesn't have any children return cursor } - var lastWorking = elemCursor - while (!elemCursor.failed) { - if (condition.apply(elemCursor.focus.get)) { - val elemModified = remaining.applyCommand(elemCursor, cmd) - // replay history of elemCursor on elemModified to make sure we're at the right position - // elemCursor = elemModified.top.get.hcursor.replay(elemCursor.history) - // todo: does this need to be re-enabled? - elemCursor = elemModified - } - lastWorking = elemCursor - elemCursor = elemCursor.right - } - val tryGoingUp = lastWorking.up - if (tryGoingUp.failed) { // try to go back up - lastWorking - } else { - tryGoingUp - } + val elemCursor2 = recurseApply(elemCursor, cmd, remaining, rewriteHistory) + elemCursor2.up.success.getOrElse(elemCursor2) } def get(cursor: ACursor, remaining: Path): Json = { diff --git a/src/main/scala/io/viash/config_mods/Value.scala b/src/main/scala/io/viash/config_mods/Value.scala index e112a006e..fd94265d5 100644 --- a/src/main/scala/io/viash/config_mods/Value.scala +++ b/src/main/scala/io/viash/config_mods/Value.scala @@ -35,13 +35,13 @@ case class JsonValue(value: Json) extends Value { // define paths case class Path(path: List[PathExp]) extends Value { - def applyCommand(json: Json, cmd: ACursor => ACursor): Json = { - applyCommand(json.hcursor, cmd).top.get + def applyCommand(json: Json, cmd: ACursor => ACursor, rewriteHistory: Boolean): Json = { + applyCommand(json.hcursor, cmd, rewriteHistory).top.get } - def applyCommand(cursor: ACursor, cmd: ACursor => ACursor): ACursor = { + def applyCommand(cursor: ACursor, cmd: ACursor => ACursor, rewriteHistory: Boolean): ACursor = { path match { case head :: tail => { - head.applyCommand(cursor, cmd, Path(tail)) + head.applyCommand(cursor, cmd, Path(tail), rewriteHistory) } case Nil => cmd(cursor) // or throw error? } diff --git a/src/main/scala/io/viash/exceptions/ConfigException.scala b/src/main/scala/io/viash/exceptions/ConfigException.scala new file mode 100644 index 000000000..98d837979 --- /dev/null +++ b/src/main/scala/io/viash/exceptions/ConfigException.scala @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.exceptions + +abstract class AbstractConfigException extends Exception { + val uri: String + val e: Throwable + val innerMessage: String + + override def getMessage(): String = e.getMessage() +} + + +case class ConfigParserException(uri: String, e: Throwable) extends AbstractConfigException { + val innerMessage: String = "invalid Viash Config content" +} + +case class ConfigYamlException(uri: String, e: Throwable) extends AbstractConfigException { + val innerMessage: String = "invalid Yaml structure" +} + +case class ConfigParserSubTypeException(tpe: String, validTypes: List[String], json: String) extends Exception { + + private val validTypesStr = validTypes.dropRight(1).mkString("'", "', '", "'") + s", and '${validTypes.last}'" + override def getMessage(): String = s"Type '$tpe' is not recognised. Valid types are $validTypesStr.\n$json" +} + +case class ConfigParserValidationException(tpe: String, json: String) extends Exception { + val shortType = tpe.split("\\.").last + override def getMessage(): String = s"Invalid data fields for $shortType.\n$json" +} \ No newline at end of file diff --git a/src/main/scala/io/viash/exceptions/ExitException.scala b/src/main/scala/io/viash/exceptions/ExitException.scala new file mode 100644 index 000000000..831e370c3 --- /dev/null +++ b/src/main/scala/io/viash/exceptions/ExitException.scala @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.exceptions + +case class ExitException(code: Int) extends Throwable diff --git a/src/main/scala/io/viash/exceptions/MalformedInputException.scala b/src/main/scala/io/viash/exceptions/MalformedInputException.scala new file mode 100644 index 000000000..60f2e3a2c --- /dev/null +++ b/src/main/scala/io/viash/exceptions/MalformedInputException.scala @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.exceptions + +case class MalformedInputException(uri: String, innerException: Throwable) extends Exception(innerException) { + override def getMessage(): String = s"Could not read config file at '$uri'.\n\tThis is likely due to a bug in the current version of Java with non-ASCII characters." +} diff --git a/src/main/scala/io/viash/helpers/MissingResourceFileException.scala b/src/main/scala/io/viash/exceptions/MissingResourceFileException.scala similarity index 97% rename from src/main/scala/io/viash/helpers/MissingResourceFileException.scala rename to src/main/scala/io/viash/exceptions/MissingResourceFileException.scala index 508af5e48..9811beaf6 100644 --- a/src/main/scala/io/viash/helpers/MissingResourceFileException.scala +++ b/src/main/scala/io/viash/exceptions/MissingResourceFileException.scala @@ -15,7 +15,7 @@ * along with this program. If not, see . */ -package io.viash.helpers +package io.viash.exceptions case class MissingResourceFileException( val resource: String, diff --git a/src/main/scala/io/viash/functionality/ArgumentGroup.scala b/src/main/scala/io/viash/functionality/ArgumentGroup.scala index 1bc507a0c..2a61725d7 100644 --- a/src/main/scala/io/viash/functionality/ArgumentGroup.scala +++ b/src/main/scala/io/viash/functionality/ArgumentGroup.scala @@ -19,9 +19,16 @@ package io.viash.functionality import io.viash.helpers.data_structures._ import io.viash.functionality.arguments.Argument +import io.viash.schemas._ +@description("A grouping of the @[arguments](argument), used to display the help message.") case class ArgumentGroup( + @description("The name of the argument group.") name: String, + + @description("Description of foo`, a description of the argument group. Multiline descriptions are supported.") description: Option[String] = None, + + @description("List of arguments.") arguments: List[Argument[_]] = Nil ) diff --git a/src/main/scala/io/viash/functionality/Author.scala b/src/main/scala/io/viash/functionality/Author.scala index 0c5c22f72..d1efc410c 100644 --- a/src/main/scala/io/viash/functionality/Author.scala +++ b/src/main/scala/io/viash/functionality/Author.scala @@ -48,14 +48,17 @@ case class Author( |* `"maintainer"`: The maintainer of the component. |* `"contributor"`: Authors who have made smaller contributions (such as code patches etc.). |""".stripMargin) + @default("Empty") roles: OneOrMore[String] = Nil, @description("Author properties. Must be a map of strings.") @deprecated("Use `info` instead.", "0.7.4", "0.8.0") + @default("Empty") props: Map[String, String] = Map.empty[String, String], @description("Structured information. Can be any shape: a string, vector, map or even nested map.") @since("Viash 0.7.4") + @default("Empty") info: Json = Json.Null ) { override def toString: String = { diff --git a/src/main/scala/io/viash/functionality/ComputationalRequirements.scala b/src/main/scala/io/viash/functionality/ComputationalRequirements.scala index b9c5f9933..190c52f1c 100644 --- a/src/main/scala/io/viash/functionality/ComputationalRequirements.scala +++ b/src/main/scala/io/viash/functionality/ComputationalRequirements.scala @@ -30,12 +30,9 @@ case class ComputationalRequirements( memory: Option[String] = None, @description("A list of commands which should be present on the system for the script to function.") @example("commands: [ which, bash, awk, date, grep, egrep, ps, sed, tail, tee ]", "yaml") + @default("Empty") commands: List[String] = Nil ) { - // START OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - @removed("Use `cpus` instead.", "0.6.1", "0.7.0") - private val n_proc: Option[Int] = None - // END OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED def memoryAsBytes: Option[BigInt] = { val Regex = "^([0-9]+) *([kmgtp]b?|b)$".r diff --git a/src/main/scala/io/viash/functionality/Functionality.scala b/src/main/scala/io/viash/functionality/Functionality.scala index 75b85f13e..6eab1d19f 100644 --- a/src/main/scala/io/viash/functionality/Functionality.scala +++ b/src/main/scala/io/viash/functionality/Functionality.scala @@ -74,6 +74,7 @@ case class Functionality( | email: tim@far.be |""".stripMargin, "yaml") @since("Viash 0.3.1") + @default("Empty") authors: List[Author] = Nil, @description( @@ -103,6 +104,7 @@ case class Functionality( | type: string |""".stripMargin, "yaml") + @default("Empty") arguments: List[Argument[_]] = Nil, @description( @@ -110,7 +112,7 @@ case class Functionality( | | - `name: foo`, the name of the argument group. | - `description: Description of foo`, a description of the argument group. Multiline descriptions are supported. - | - `arguments: [arg1, arg2, ...]`, list of the arguments names. + | - `arguments: [arg1, arg2, ...]`, list of the arguments. | |""".stripMargin) @example( @@ -154,6 +156,7 @@ case class Functionality( "bash", "This results in the following output when calling the component with the `--help` argument:") @since("Viash 0.5.14") + @default("Empty") argument_groups: List[ArgumentGroup] = Nil, @description( @@ -175,6 +178,7 @@ case class Functionality( | path: resource1.txt |""".stripMargin, "yaml") + @default("Empty") resources: List[Resource] = Nil, @description("A description of the component. This will be displayed with `--help`.") @@ -200,6 +204,7 @@ case class Functionality( | - path: resource1.txt |""".stripMargin, "yaml") + @default("Empty") test_resources: List[Resource] = Nil, @description("Structured information. Can be any shape: a string, vector, map or even nested map.") @@ -208,10 +213,12 @@ case class Functionality( | twitter: wizzkid | classes: [ one, two, three ]""".stripMargin, "yaml") @since("Viash 0.4.0") + @default("Empty") info: Json = Json.Null, @description("Allows setting a component to active, deprecated or disabled.") @since("Viash 0.6.0") + @default("Enabled") status: Status = Status.Enabled, @description( @@ -226,6 +233,7 @@ case class Functionality( |""".stripMargin, "yaml") @since("Viash 0.6.0") + @default("Empty") requirements: ComputationalRequirements = ComputationalRequirements(), // The variables below are for internal use and shouldn't be publicly documented @@ -236,110 +244,29 @@ case class Functionality( @internalFunctionality set_wd_to_resources_dir: Boolean = false ) { - // START OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - @description("Adds the resources directory to the PATH variable when set to true. This is set to false by default.") - @since("Viash 0.5.5") - @removed("Extending the PATH turned out to be not desirable.", "", "0.5.11") - private val add_resources_to_path: Boolean = false - - @description("One or more Bash/R/Python scripts to be used to test the component behaviour when `viash test` is invoked. Additional files of type `file` will be made available only during testing. Each test script should expect no command-line inputs, be platform-independent, and return an exit code >0 when unexpected behaviour occurs during testing.") - @removed("Use `test_resources` instead. No functional difference.", "0.5.13", "0.7.0") - private val tests: List[Resource] = Nil - - @description("Setting this to false with disable this component when using namespaces.") - @since("Viash 0.5.13") - @removed("Use `status` instead.", "0.6.0", "0.7.0") - private val enabled: Boolean = true - - @description("A list of input arguments in addition to the `arguments` list. Any arguments specified here will have their `type` set to `file` and the `direction` set to `input` by default.") - @example( - """inputs: - | - name: input_file - | - name: another_input""".stripMargin, - "yaml") - @exampleWithDescription( - """component_with_inputs - | - | Inputs: - | input_file - | type: file - | - | another_input - | type: file""".stripMargin, - "bash", - "This results in the following output when calling the component with the `--help` argument:") - @since("Viash 0.5.11") - @removed("Use `arguments` instead.", "0.6.0", "0.7.0") - private val inputs: List[Argument[_]] = Nil - - @description("A list of output arguments in addition to the `arguments` list. Any arguments specified here will have their `type` set to `file` and thr `direction` set to `output` by default.") - @example( - """outputs: - | - name: output_file - | - name: another_output""".stripMargin, - "yaml") - @exampleWithDescription( - """component_with_outputs - | - | Outputs: - | output_file - | type: file, output - | - | another_output - | type: file, output""".stripMargin, - "bash", - "This results in the following output when calling the component with the `--help` argument:") - @since("Viash 0.5.11") - @removed("Use `arguments` instead.", "0.6.0", "0.7.0") - private val outputs: List[Argument[_]] = Nil - // END OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED // Combine inputs, outputs and arguments into one combined list def allArguments = arguments ::: argument_groups.flatMap(arg => arg.arguments) - private def addToArgGroup(argumentGroups: List[ArgumentGroup], name: String, arguments: List[Argument[_]]): Option[ArgumentGroup] = { - // Check whether an argument group of 'name' exists. - val existing = argumentGroups.find(gr => name == gr.name) - - // if there are no arguments missing from the argument group, just return the existing group (if any) + def allArgumentGroups: List[ArgumentGroup] = { if (arguments.isEmpty) { - existing - - // if there are missing arguments and there is an existing group, add the missing arguments to it - } else if (existing.isDefined) { - Some(existing.get.copy( - arguments = existing.get.arguments.toList ::: arguments - )) - - // else create a new group + // if there are no arguments, just return the argument groups as is + argument_groups + } else if (argument_groups.exists(_.name == "Arguments")) { + // if there is already an argument group named 'Arguments', extend it with the arguments + argument_groups.map{ + case gr if gr.name == "Arguments" => + gr.copy(arguments = gr.arguments ::: arguments) + case gr => gr + } } else { - Some(ArgumentGroup( - name = name, + // else create a new argument group + argument_groups ::: List(ArgumentGroup( + name = "Arguments", arguments = arguments )) } } - - def allArgumentGroups: List[ArgumentGroup] = { - val joinGroup = Map("Inputs" -> inputs, "Outputs" -> outputs, "Arguments" -> arguments) - val groups = argument_groups.map{gr => - if (List("Inputs", "Outputs", "Arguments") contains (gr.name)) { - addToArgGroup(argument_groups, gr.name, joinGroup(gr.name)).get - } else { - gr - } - } - val missingGroups = joinGroup.flatMap{ - case (name, args) => - if (!groups.exists(_.name == name)) { - addToArgGroup(argument_groups, name, args) - } else { - None - } - }.toList - - missingGroups ::: groups - } // check whether there are not multiple positional arguments with multiplicity >1 // and if there is one, whether its position is last diff --git a/src/main/scala/io/viash/functionality/arguments/Argument.scala b/src/main/scala/io/viash/functionality/arguments/Argument.scala index cbe49fe9a..bc232a4e3 100644 --- a/src/main/scala/io/viash/functionality/arguments/Argument.scala +++ b/src/main/scala/io/viash/functionality/arguments/Argument.scala @@ -49,6 +49,14 @@ import java.nio.file.Paths | type: string |""".stripMargin, "yaml") +@subclass("BooleanArgument") +@subclass("BooleanTrueArgument") +@subclass("BooleanFalseArgument") +@subclass("DoubleArgument") +@subclass("FileArgument") +@subclass("IntegerArgument") +@subclass("LongArgument") +@subclass("StringArgument") abstract class Argument[Type] { @description("Specifies the type of the argument.") val `type`: String diff --git a/src/main/scala/io/viash/functionality/arguments/BooleanArgument.scala b/src/main/scala/io/viash/functionality/arguments/BooleanArgument.scala index a171853e6..d2627c7c5 100644 --- a/src/main/scala/io/viash/functionality/arguments/BooleanArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/BooleanArgument.scala @@ -35,6 +35,7 @@ abstract class BooleanArgumentBase extends Argument[Boolean] { | alternatives: ["-t"] |""".stripMargin, "yaml") +@subclass("boolean") case class BooleanArgument( @description( """The name of the argument. Can be in the formats `--trim`, `-t` or `trim`. The number of dashes determines how values can be passed: @@ -46,6 +47,7 @@ case class BooleanArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -57,6 +59,7 @@ case class BooleanArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @description("An example value for this argument. If no [`default`](#default) property was specified, this will be used for that purpose.") @@ -66,6 +69,7 @@ case class BooleanArgument( | example: true |""".stripMargin, "yaml") + @default("Empty") example: OneOrMore[Boolean] = Nil, @description("The default value when no argument value is provided. This will not work if the [`required`](#required) property is enabled.") @@ -74,7 +78,8 @@ case class BooleanArgument( | type: boolean | default: true |""".stripMargin, - "yaml") + "yaml") + @default("Empty") default: OneOrMore[Boolean] = Nil, @description("Make the value for this argument required. If set to `true`, an error will be produced if no value was provided. `false` by default.") @@ -84,6 +89,7 @@ case class BooleanArgument( | required: true |""".stripMargin, "yaml") + @default("False") required: Boolean = false, @undocumented @@ -97,6 +103,7 @@ case class BooleanArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_boolean=true:true:false", "bash", "Here's an example of how to use this:") + @default("False") multiple: Boolean = false, @description("The delimiter character for providing [`multiple`](#multiple) values. `:` by default.") @@ -108,6 +115,7 @@ case class BooleanArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_boolean=true,true,false", "bash", "Here's an example of how to use this:") + @default(":") multiple_sep: String = ":", @undocumented @@ -144,6 +152,7 @@ case class BooleanArgument( | alternatives: ["-s"] |""".stripMargin, "yaml") +@subclass("boolean_true") case class BooleanTrueArgument( @description( """The name of the argument. Can be in the formats `--silent`, `-s` or `silent`. The number of dashes determines how values can be passed: @@ -155,6 +164,7 @@ case class BooleanTrueArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -166,6 +176,7 @@ case class BooleanTrueArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @undocumented @@ -214,6 +225,7 @@ case class BooleanTrueArgument( | alternatives: ["-nl"] |""".stripMargin, "yaml") +@subclass("boolean_false") case class BooleanFalseArgument( @description( """The name of the argument. Can be in the formats `--no-log`, `-n` or `no-log`. The number of dashes determines how values can be passed: @@ -225,6 +237,7 @@ case class BooleanFalseArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -236,6 +249,7 @@ case class BooleanFalseArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @undocumented diff --git a/src/main/scala/io/viash/functionality/arguments/DoubleArgument.scala b/src/main/scala/io/viash/functionality/arguments/DoubleArgument.scala index c92c8bccf..9815adfe7 100644 --- a/src/main/scala/io/viash/functionality/arguments/DoubleArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/DoubleArgument.scala @@ -31,6 +31,7 @@ import io.viash.schemas._ | alternatives: ["-l"] |""".stripMargin, "yaml") +@subclass("double") case class DoubleArgument( @description( """The name of the argument. Can be in the formats `--foo`, `-f` or `foo`. The number of dashes determines how values can be passed: @@ -42,6 +43,7 @@ case class DoubleArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -53,6 +55,7 @@ case class DoubleArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @description("An example value for this argument. If no [`default`](#default) property was specified, this will be used for that purpose.") @@ -62,6 +65,7 @@ case class DoubleArgument( | example: 5.8 |""".stripMargin, "yaml") + @default("Empty") example: OneOrMore[Double] = Nil, @description("The default value when no argument value is provided. This will not work if the [`required`](#required) property is enabled.") @@ -71,6 +75,7 @@ case class DoubleArgument( | default: 5.8 |""".stripMargin, "yaml") + @default("Empty") default: OneOrMore[Double] = Nil, @description("Make the value for this argument required. If set to `true`, an error will be produced if no value was provided. `false` by default.") @@ -80,6 +85,7 @@ case class DoubleArgument( | required: true |""".stripMargin, "yaml") + @default("False") required: Boolean = false, @description("Minimum allowed value for this argument. If set and the provided value is lower than the minimum, an error will be produced. Can be combined with [`max`](#max) to clamp values.") @@ -111,6 +117,7 @@ case class DoubleArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_double=5.8:22.6:200.4", "bash", "Here's an example of how to use this:") + @default("False") multiple: Boolean = false, @description("The delimiter character for providing [`multiple`](#multiple) values. `:` by default.") @@ -122,6 +129,7 @@ case class DoubleArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_double=5.8,22.6,200.4", "bash", "Here's an example of how to use this:") + @default(":") multiple_sep: String = ":", dest: String = "par", diff --git a/src/main/scala/io/viash/functionality/arguments/FileArgument.scala b/src/main/scala/io/viash/functionality/arguments/FileArgument.scala index 1931ec0fe..1a917896a 100644 --- a/src/main/scala/io/viash/functionality/arguments/FileArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/FileArgument.scala @@ -32,6 +32,7 @@ import io.viash.schemas._ | alternatives: ["-i"] |""".stripMargin, "yaml") +@subclass("file") case class FileArgument( @description( """The name of the argument. Can be in the formats `--foo`, `-f` or `foo`. The number of dashes determines how values can be passed: @@ -43,6 +44,7 @@ case class FileArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -54,6 +56,7 @@ case class FileArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @description("An example value for this argument. If no [`default`](#default) property was specified, this will be used for that purpose.") @@ -63,6 +66,7 @@ case class FileArgument( | example: data.csv |""".stripMargin, "yaml") + @default("Empty") example: OneOrMore[Path] = Nil, @description("The default value when no argument value is provided. This will not work if the [`required`](#required) property is enabled.") @@ -72,6 +76,7 @@ case class FileArgument( | default: data.csv |""".stripMargin, "yaml") + @default("Empty") default: OneOrMore[Path] = Nil, @description("Checks whether the file or folder exists. For input files, this check will happen " + @@ -82,6 +87,7 @@ case class FileArgument( | must_exist: true |""".stripMargin, "yaml") + @default("True") must_exist: Boolean = true, @description("If the output filename is a path and it does not exist, create it before executing the script (only for `direction: output`).") @@ -92,6 +98,7 @@ case class FileArgument( | create_parent: true |""".stripMargin, "yaml") + @default("True") create_parent: Boolean = true, @description("Make the value for this argument required. If set to `true`, an error will be produced if no value was provided. `false` by default.") @@ -101,6 +108,7 @@ case class FileArgument( | required: true |""".stripMargin, "yaml") + @default("False") required: Boolean = false, @description("Makes this argument an `input` or an `output`, as in does the file/folder needs to be read or written. `input` by default.") @@ -110,6 +118,7 @@ case class FileArgument( | direction: output |""".stripMargin, "yaml") + @default("Input") direction: Direction = Input, @description("Treat the argument value as an array. Arrays can be passed using the delimiter `--foo=1:2:3` or by providing the same argument multiple times `--foo 1 --foo 2`. You can use a custom delimiter by using the [`multiple_sep`](#multiple_sep) property. `false` by default.") @@ -120,6 +129,7 @@ case class FileArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_files=firstFile.csv:anotherFile.csv:yetAnother.csv", "bash", "Here's an example of how to use this:") + @default("False") multiple: Boolean = false, @description("The delimiter character for providing [`multiple`](#multiple) values. `:` by default.") @@ -131,6 +141,7 @@ case class FileArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_files=firstFile.csv,anotherFile.csv,yetAnother.csv", "bash", "Here's an example of how to use this:") + @default(":") multiple_sep: String = ":", dest: String = "par", diff --git a/src/main/scala/io/viash/functionality/arguments/IntegerArgument.scala b/src/main/scala/io/viash/functionality/arguments/IntegerArgument.scala index ac593d807..140ef5360 100644 --- a/src/main/scala/io/viash/functionality/arguments/IntegerArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/IntegerArgument.scala @@ -31,6 +31,7 @@ import io.viash.schemas._ | alternatives: ["-c"] |""".stripMargin, "yaml") +@subclass("integer") case class IntegerArgument( @description( """The name of the argument. Can be in the formats `--foo`, `-f` or `foo`. The number of dashes determines how values can be passed: @@ -42,6 +43,7 @@ case class IntegerArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -53,6 +55,7 @@ case class IntegerArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @description("An example value for this argument. If no [`default`](#default) property was specified, this will be used for that purpose.") @@ -62,6 +65,7 @@ case class IntegerArgument( | example: 100 |""".stripMargin, "yaml") + @default("Empty") example: OneOrMore[Int] = Nil, @description("The default value when no argument value is provided. This will not work if the [`required`](#required) property is enabled.") @@ -71,6 +75,7 @@ case class IntegerArgument( | default: 100 |""".stripMargin, "yaml") + @default("Empty") default: OneOrMore[Int] = Nil, @description("Make the value for this argument required. If set to `true`, an error will be produced if no value was provided. `false` by default.") @@ -80,6 +85,7 @@ case class IntegerArgument( | required: true |""".stripMargin, "yaml") + @default("False") required: Boolean = false, @description("Limit the amount of valid values for this argument to those set in this list. When set and a value not present in the list is provided, an error will be produced.") @@ -89,6 +95,7 @@ case class IntegerArgument( | choices: [1024, 2048, 4096] |""".stripMargin, "yaml") + @default("Empty") choices: List[Int] = Nil, @description("Minimum allowed value for this argument. If set and the provided value is lower than the minimum, an error will be produced. Can be combined with [`max`](#max) to clamp values.") @@ -120,6 +127,7 @@ case class IntegerArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:") + @default("False") multiple: Boolean = false, @description("The delimiter character for providing [`multiple`](#multiple) values. `:` by default.") @@ -131,6 +139,7 @@ case class IntegerArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_integer=10:80:152", "bash", "Here's an example of how to use this:") + @default(":") multiple_sep: String = ":", dest: String = "par", diff --git a/src/main/scala/io/viash/functionality/arguments/LongArgument.scala b/src/main/scala/io/viash/functionality/arguments/LongArgument.scala index 2a87fbf67..13d32c37a 100644 --- a/src/main/scala/io/viash/functionality/arguments/LongArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/LongArgument.scala @@ -32,6 +32,7 @@ import io.viash.schemas._ |""".stripMargin, "yaml") @since("Viash 0.6.1") +@subclass("long") case class LongArgument( @description( """The name of the argument. Can be in the formats `--foo`, `-f` or `foo`. The number of dashes determines how values can be passed: @@ -43,6 +44,7 @@ case class LongArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") @@ -54,6 +56,7 @@ case class LongArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @description("An example value for this argument. If no [`default`](#default) property was specified, this will be used for that purpose.") @@ -63,6 +66,7 @@ case class LongArgument( | example: 100 |""".stripMargin, "yaml") + @default("Empty") example: OneOrMore[Long] = Nil, @description("The default value when no argument value is provided. This will not work if the [`required`](#required) property is enabled.") @@ -72,6 +76,7 @@ case class LongArgument( | default: 100 |""".stripMargin, "yaml") + @default("Empty") default: OneOrMore[Long] = Nil, @description("Make the value for this argument required. If set to `true`, an error will be produced if no value was provided. `false` by default.") @@ -81,6 +86,7 @@ case class LongArgument( | required: true |""".stripMargin, "yaml") + @default("False") required: Boolean = false, @description("Limit the amount of valid values for this argument to those set in this list. When set and a value not present in the list is provided, an error will be produced.") @@ -90,6 +96,7 @@ case class LongArgument( | choices: [1024, 2048, 4096] |""".stripMargin, "yaml") + @default("Empty") choices: List[Long] = Nil, @description("Minimum allowed value for this argument. If set and the provided value is lower than the minimum, an error will be produced. Can be combined with [`max`](#max) to clamp values.") @@ -121,6 +128,7 @@ case class LongArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:") + @default("False") multiple: Boolean = false, @description("The delimiter character for providing [`multiple`](#multiple) values. `:` by default.") @@ -132,6 +140,7 @@ case class LongArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_long=10:80:152", "bash", "Here's an example of how to use this:") + @default(":") multiple_sep: String = ":", dest: String = "par", diff --git a/src/main/scala/io/viash/functionality/arguments/StringArgument.scala b/src/main/scala/io/viash/functionality/arguments/StringArgument.scala index 4c818604a..da86a9d39 100644 --- a/src/main/scala/io/viash/functionality/arguments/StringArgument.scala +++ b/src/main/scala/io/viash/functionality/arguments/StringArgument.scala @@ -31,6 +31,7 @@ import io.viash.schemas._ | alternatives: ["-q"] |""".stripMargin, "yaml") +@subclass("string") case class StringArgument( @description( """The name of the argument. Can be in the formats `--foo`, `-f` or `foo`. The number of dashes determines how values can be passed: @@ -42,9 +43,11 @@ case class StringArgument( name: String, @description("List of alternative format variations for this argument.") + @default("Empty") alternatives: OneOrMore[String] = Nil, @description("A description of the argument. This will be displayed with `--help`.") + @default("Empty") description: Option[String] = None, @description("Structured information. Can be any shape: a string, vector, map or even nested map.") @@ -53,6 +56,7 @@ case class StringArgument( | category: cat1 | labels: [one, two, three]""".stripMargin, "yaml") @since("Viash 0.6.3") + @default("Empty") info: Json = Json.Null, @description("An example value for this argument. If no [`default`](#default) property was specified, this will be used for that purpose.") @@ -62,6 +66,7 @@ case class StringArgument( | example: "Hello World" |""".stripMargin, "yaml") + @default("Empty") example: OneOrMore[String] = Nil, @description("The default value when no argument value is provided. This will not work if the [`required`](#required) property is enabled.") @@ -71,6 +76,7 @@ case class StringArgument( | default: "The answer is 42" |""".stripMargin, "yaml") + @default("Empty") default: OneOrMore[String] = Nil, @description("Make the value for this argument required. If set to `true`, an error will be produced if no value was provided. `false` by default.") @@ -80,6 +86,7 @@ case class StringArgument( | required: true |""".stripMargin, "yaml") + @default("Empty") required: Boolean = false, @description("Limit the amount of valid values for this argument to those set in this list. When set and a value not present in the list is provided, an error will be produced.") @@ -89,6 +96,7 @@ case class StringArgument( | choices: ["python", "r", "javascript"] |""".stripMargin, "yaml") + @default("Empty") choices: List[String] = Nil, @undocumented @@ -102,6 +110,7 @@ case class StringArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_string=Marc:Susan:Paul", "bash", "Here's an example of how to use this:") + @default("False") multiple: Boolean = false, @description("The delimiter character for providing [`multiple`](#multiple) values. `:` by default.") @@ -113,6 +122,7 @@ case class StringArgument( |""".stripMargin, "yaml") @exampleWithDescription("my_component --my_string=Marc,Susan,Paul", "bash", "Here's an example of how to use this:") + @default(":") multiple_sep: String = ":", dest: String = "par", diff --git a/src/main/scala/io/viash/functionality/arguments/package.scala b/src/main/scala/io/viash/functionality/arguments/package.scala index 1a8ad2450..48738c042 100644 --- a/src/main/scala/io/viash/functionality/arguments/package.scala +++ b/src/main/scala/io/viash/functionality/arguments/package.scala @@ -20,6 +20,9 @@ package io.viash.functionality import io.circe.{Decoder, Encoder, Json} import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import cats.syntax.functor._ // for .widen +import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ +import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ +import io.viash.exceptions.ConfigParserSubTypeException package object arguments { @@ -37,9 +40,9 @@ package object arguments { io.circe.Decoder.decodeDouble or Decoder.instance { cursor => cursor.value.as[String].map(_.toLowerCase()) match { - case Right("+inf" | "+infinity" | "positiveinfinity" | "positiveinf") => Right(Double.PositiveInfinity) - case Right("-inf" | "-infinity" | "negativeinfinity" | "negativeinf") => Right(Double.NegativeInfinity) - case Right("nan") => Right(Double.NaN) + case Right(".inf" | "+.inf" | "+inf" | "+infinity" | "positiveinfinity" | "positiveinf") => Right(Double.PositiveInfinity) + case Right("-.inf" | "-inf" | "-infinity" | "negativeinfinity" | "negativeinf") => Right(Double.NegativeInfinity) + case Right(".nan" | "nan") => Right(Double.NaN) case a => a.map(_.toDouble) } } @@ -84,14 +87,14 @@ package object arguments { objJson deepMerge typeJson } - implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoder - implicit val decodeIntegerArgument: Decoder[IntegerArgument] = deriveConfiguredDecoder - implicit val decodeLongArgument: Decoder[LongArgument] = deriveConfiguredDecoder - implicit val decodeDoubleArgument: Decoder[DoubleArgument] = deriveConfiguredDecoder - implicit val decodeBooleanArgumentR: Decoder[BooleanArgument] = deriveConfiguredDecoder - implicit val decodeBooleanArgumentT: Decoder[BooleanTrueArgument] = deriveConfiguredDecoder - implicit val decodeBooleanArgumentF: Decoder[BooleanFalseArgument] = deriveConfiguredDecoder - implicit val decodeFileArgument: Decoder[FileArgument] = deriveConfiguredDecoder + implicit val decodeStringArgument: Decoder[StringArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeIntegerArgument: Decoder[IntegerArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeLongArgument: Decoder[LongArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeDoubleArgument: Decoder[DoubleArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeBooleanArgumentR: Decoder[BooleanArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeBooleanArgumentT: Decoder[BooleanTrueArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeBooleanArgumentF: Decoder[BooleanFalseArgument] = deriveConfiguredDecoderFullChecks + implicit val decodeFileArgument: Decoder[FileArgument] = deriveConfiguredDecoderFullChecks implicit def decodeDataArgument: Decoder[Argument[_]] = Decoder.instance { cursor => @@ -105,7 +108,7 @@ package object arguments { case Right("boolean_true") => decodeBooleanArgumentT.widen case Right("boolean_false") => decodeBooleanArgumentF.widen case Right("file") => decodeFileArgument.widen - case Right(typ) => throw new RuntimeException("Type " + typ + " is not recognised. Valid types are string, integer, long, double, boolean, boolean_true, boolean_false and file.") + case Right(typ) => invalidSubTypeDecoder[StringArgument](typ, List("string", "integer", "long", "double", "boolean", "boolean_true", "boolean_false", "file")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/functionality/package.scala b/src/main/scala/io/viash/functionality/package.scala index ba684a5a3..1ca60a2d7 100644 --- a/src/main/scala/io/viash/functionality/package.scala +++ b/src/main/scala/io/viash/functionality/package.scala @@ -21,56 +21,65 @@ import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfigur import io.circe.{Decoder, Encoder, Json} import io.circe.ACursor -package object functionality { +import io.viash.helpers.Logging + +package object functionality extends Logging { // import implicits import functionality.arguments._ import functionality.resources._ import functionality.Status._ import io.viash.helpers.circe._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ // encoder and decoder for Functionality implicit val encodeFunctionality: Encoder.AsObject[Functionality] = deriveConfiguredEncoder // add file & direction defaults for inputs & outputs - implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeFunctionality: Decoder[Functionality] = deriveConfiguredDecoderFullChecks // encoder and decoder for Author implicit val encodeAuthor: Encoder.AsObject[Author] = deriveConfiguredEncoder - implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeAuthor: Decoder[Author] = deriveConfiguredDecoderFullChecks // encoder and decoder for Requirements implicit val encodeComputationalRequirements: Encoder.AsObject[ComputationalRequirements] = deriveConfiguredEncoder - implicit val decodeComputationalRequirements: Decoder[ComputationalRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeComputationalRequirements: Decoder[ComputationalRequirements] = deriveConfiguredDecoderFullChecks // encoder and decoder for ArgumentGroup implicit val encodeArgumentGroup: Encoder.AsObject[ArgumentGroup] = deriveConfiguredEncoder - implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoder[ArgumentGroup].prepare { - checkDeprecation[ArgumentGroup](_) // check for deprecations - .withFocus(_.mapObject{ ag0 => + implicit val decodeArgumentGroup: Decoder[ArgumentGroup] = deriveConfiguredDecoder[ArgumentGroup] + .validate( + validator[ArgumentGroup], + s"Could not convert json to ArgumentGroup." + ) + .prepare { + checkDeprecation[ArgumentGroup](_) // check for deprecations + .withFocus(_.mapObject{ ag0 => - // Check whether arguments contains a string value instead of an object. The support for this was removed in Viash 0.7.0 - ag0.apply("arguments") match { - case Some(args) => - args.mapArray(argVector => { - for (arg <- argVector) { - if (arg.isString) { - Console.err.println( - s"""Error: specifying strings in the .argument field of argument group '${ag0.apply("name").get.asString.get}' was removed. - |The .arguments field of an argument group should only contain arguments. - |To solve this issue, copy the argument ${arg} directly into the argument group.""".stripMargin) + // Check whether arguments contains a string value instead of an object. The support for this was removed in Viash 0.7.0 + ag0.apply("arguments") match { + case Some(args) => + args.mapArray(argVector => { + for (arg <- argVector) { + if (arg.isString) { + info( + s"""Error: specifying strings in the .argument field of argument group '${ag0.apply("name").get.asString.get}' was removed. + |The .arguments field of an argument group should only contain arguments. + |To solve this issue, copy the argument ${arg} directly into the argument group.""".stripMargin) + } } - } - argVector - }) - case _ => None - } + argVector + }) + case _ => None + } - ag0 + ag0 + } + ) } - ) - } // encoder and decoder for Status, make string lowercase before decoding diff --git a/src/main/scala/io/viash/functionality/resources/BashScript.scala b/src/main/scala/io/viash/functionality/resources/BashScript.scala index 351d3a786..22417b62d 100644 --- a/src/main/scala/io/viash/functionality/resources/BashScript.scala +++ b/src/main/scala/io/viash/functionality/resources/BashScript.scala @@ -28,6 +28,7 @@ import io.viash.helpers.Bash @description("""An executable Bash script. |When defined in functionality.resources, only the first entry will be executed when running the built component or when running `viash run`. |When defined in functionality.test_resources, all entries will be executed during `viash test`.""".stripMargin) +@subclass("bash_script") case class BashScript( path: Option[String] = None, text: Option[String] = None, @@ -51,18 +52,11 @@ case class BashScript( val paramsCode = parSet.mkString("\n") + "\n" ScriptInjectionMods(params = paramsCode) } - - def command(script: String): String = { - "bash \"" + script + "\"" - } - - def commandSeq(script: String): Seq[String] = { - Seq("bash", script) - } } object BashScript extends ScriptCompanion { val commentStr = "#" val extension = "sh" val `type` = "bash_script" + val executor = Seq("bash") } diff --git a/src/main/scala/io/viash/functionality/resources/CSharpScript.scala b/src/main/scala/io/viash/functionality/resources/CSharpScript.scala index 267a28791..d631c0d3d 100644 --- a/src/main/scala/io/viash/functionality/resources/CSharpScript.scala +++ b/src/main/scala/io/viash/functionality/resources/CSharpScript.scala @@ -28,6 +28,7 @@ import io.viash.helpers.Bash @description("""An executable C# script. |When defined in functionality.resources, only the first entry will be executed when running the built component or when running `viash run`. |When defined in functionality.test_resources, all entries will be executed during `viash test`.""".stripMargin) +@subclass("csharp_script") case class CSharpScript( path: Option[String] = None, text: Option[String] = None, @@ -107,19 +108,12 @@ case class CSharpScript( ScriptInjectionMods(params = paramsCode.mkString) } - - def command(script: String): String = { - "dotnet script \"" + script + "\"" - } - - def commandSeq(script: String): Seq[String] = { - Seq("dotnet", "script", script) - } } object CSharpScript extends ScriptCompanion { val commentStr = "//" val extension = "csx" val `type` = "csharp_script" + val executor = Seq("dotnet", "script") } diff --git a/src/main/scala/io/viash/functionality/resources/Executable.scala b/src/main/scala/io/viash/functionality/resources/Executable.scala index 5579d4842..a458b7217 100644 --- a/src/main/scala/io/viash/functionality/resources/Executable.scala +++ b/src/main/scala/io/viash/functionality/resources/Executable.scala @@ -25,6 +25,7 @@ import io.viash.functionality.arguments.Argument import io.viash.schemas._ @description("An executable file.") +@subclass("executable") case class Executable( path: Option[String] = None, text: Option[String] = None, @@ -46,17 +47,12 @@ case class Executable( override def write(path: Path, overwrite: Boolean): Unit = {} - def command(script: String): String = { - script - } - - def commandSeq(script: String): Seq[String] = { - Seq(script) - } + override def command(script: String): String = script } object Executable extends ScriptCompanion { val commentStr = "#" val extension = "*" val `type` = "executable" + val executor = Nil } diff --git a/src/main/scala/io/viash/functionality/resources/JavaScriptScript.scala b/src/main/scala/io/viash/functionality/resources/JavaScriptScript.scala index e5b45b0a8..4f00cb568 100644 --- a/src/main/scala/io/viash/functionality/resources/JavaScriptScript.scala +++ b/src/main/scala/io/viash/functionality/resources/JavaScriptScript.scala @@ -28,6 +28,7 @@ import io.viash.helpers.Bash @description("""An executable JavaScript script. |When defined in functionality.resources, only the first entry will be executed when running the built component or when running `viash run`. |When defined in functionality.test_resources, all entries will be executed during `viash test`.""".stripMargin) +@subclass("javascript_script") case class JavaScriptScript( path: Option[String] = None, text: Option[String] = None, @@ -81,18 +82,11 @@ case class JavaScriptScript( } ScriptInjectionMods(params = paramsCode.mkString) } - - def command(script: String): String = { - "node \"" + script + "\"" - } - - def commandSeq(script: String): Seq[String] = { - Seq("node", script) - } } object JavaScriptScript extends ScriptCompanion { val commentStr = "//" val extension = "js" val `type` = "javascript_script" + val executor = Seq("node") } diff --git a/src/main/scala/io/viash/functionality/resources/NextflowScript.scala b/src/main/scala/io/viash/functionality/resources/NextflowScript.scala index 39d363a32..1603ff238 100644 --- a/src/main/scala/io/viash/functionality/resources/NextflowScript.scala +++ b/src/main/scala/io/viash/functionality/resources/NextflowScript.scala @@ -24,6 +24,7 @@ import java.net.URI import io.viash.functionality.arguments.Argument @description("""A Nextflow script. Work in progress; added mainly for annotation at the moment.""".stripMargin) +@subclass("nextflow_script") case class NextflowScript( path: Option[String] = None, text: Option[String] = None, @@ -48,21 +49,20 @@ case class NextflowScript( ScriptInjectionMods() } - def command(script: String): String = { + override def command(script: String): String = { val entryStr = entrypoint match { case Some(entry) => " -entry " + entry case None => "" } - "nextflow run . -main-script \"" + script + "\"" + entryStr + super.command(script) + entryStr } - def commandSeq(script: String): Seq[String] = { + override def commandSeq(script: String): Seq[String] = { val entrySeq = entrypoint match { case Some(entry) => Seq("-entry", entry) case None => Seq() } - // Seq("nextflow", "run", script) ++ entrySeq - Seq("nextflow", "run", ".", "-main-script", script) ++ entrySeq + super.commandSeq(script) ++ entrySeq } } @@ -70,4 +70,5 @@ object NextflowScript extends ScriptCompanion { val commentStr = "//" val extension = "nf" val `type` = "nextflow_script" + val executor = Seq("nextflow", "run", ".", "-main-script") } \ No newline at end of file diff --git a/src/main/scala/io/viash/functionality/resources/PlainFile.scala b/src/main/scala/io/viash/functionality/resources/PlainFile.scala index 57bff7fc8..da1c181c5 100644 --- a/src/main/scala/io/viash/functionality/resources/PlainFile.scala +++ b/src/main/scala/io/viash/functionality/resources/PlainFile.scala @@ -22,6 +22,7 @@ import io.viash.schemas._ import java.net.URI @description("""A plain file. This can only be used as a supporting resource for the main script or unit tests.""") +@subclass("file") case class PlainFile( path: Option[String] = None, text: Option[String] = None, diff --git a/src/main/scala/io/viash/functionality/resources/PythonScript.scala b/src/main/scala/io/viash/functionality/resources/PythonScript.scala index 838ce74d4..d65aecd2d 100644 --- a/src/main/scala/io/viash/functionality/resources/PythonScript.scala +++ b/src/main/scala/io/viash/functionality/resources/PythonScript.scala @@ -28,6 +28,7 @@ import io.viash.schemas._ @description("""An executable Python script. |When defined in functionality.resources, only the first entry will be executed when running the built component or when running `viash run`. |When defined in functionality.test_resources, all entries will be executed during `viash test`.""".stripMargin) +@subclass("python_script") case class PythonScript( path: Option[String] = None, text: Option[String] = None, @@ -83,18 +84,13 @@ case class PythonScript( ScriptInjectionMods(params = paramsCode.mkString) } - - def command(script: String): String = { - "python \"" + script + "\"" - } - - def commandSeq(script: String): Seq[String] = { - Seq("python", script) - } } object PythonScript extends ScriptCompanion { val commentStr = "#" val extension = "py" val `type` = "python_script" + // The -B argument stops Python from creating .pyc or .pyo files + // on importing functions from other files. + val executor = Seq("python", "-B") } \ No newline at end of file diff --git a/src/main/scala/io/viash/functionality/resources/RScript.scala b/src/main/scala/io/viash/functionality/resources/RScript.scala index dd975d720..507cac587 100644 --- a/src/main/scala/io/viash/functionality/resources/RScript.scala +++ b/src/main/scala/io/viash/functionality/resources/RScript.scala @@ -28,6 +28,7 @@ import java.net.URI @description("""An executable R script. |When defined in functionality.resources, only the first entry will be executed when running the built component or when running `viash run`. |When defined in functionality.test_resources, all entries will be executed during `viash test`.""".stripMargin) +@subclass("r_script") case class RScript( path: Option[String] = None, text: Option[String] = None, @@ -93,18 +94,11 @@ case class RScript( |""".stripMargin ScriptInjectionMods(params = outCode) } - - def command(script: String): String = { - "Rscript \"" + script + "\"" - } - - def commandSeq(script: String): Seq[String] = { - Seq("Rscript", script) - } } object RScript extends ScriptCompanion { val commentStr = "#" val extension = "R" val `type` = "r_script" + val executor = Seq("Rscript") } \ No newline at end of file diff --git a/src/main/scala/io/viash/functionality/resources/Resource.scala b/src/main/scala/io/viash/functionality/resources/Resource.scala index 239f46225..a13bcab89 100644 --- a/src/main/scala/io/viash/functionality/resources/Resource.scala +++ b/src/main/scala/io/viash/functionality/resources/Resource.scala @@ -19,7 +19,8 @@ package io.viash.functionality.resources import java.net.URI -import io.viash.helpers.{IO, MissingResourceFileException} +import io.viash.helpers.IO +import io.viash.exceptions.MissingResourceFileException import java.nio.file.{Path, Paths} import java.nio.file.NoSuchFileException import io.viash.schemas._ @@ -43,6 +44,15 @@ import io.viash.schemas._ | path: resource1.txt |""".stripMargin, "yaml") +@subclass("BashScript") +@subclass("CSharpScript") +@subclass("Executable") +@subclass("JavaScriptScript") +@subclass("NextflowScript") +@subclass("PlainFile") +@subclass("PythonScript") +@subclass("RScript") +@subclass("ScalaScript") trait Resource { @description("Specifies the type of the resource. The first resource cannot be of type `file`. When the type is not specified, the default type is simply `file`.") val `type`: String diff --git a/src/main/scala/io/viash/functionality/resources/ScalaScript.scala b/src/main/scala/io/viash/functionality/resources/ScalaScript.scala index 7f10673a3..725a325f3 100644 --- a/src/main/scala/io/viash/functionality/resources/ScalaScript.scala +++ b/src/main/scala/io/viash/functionality/resources/ScalaScript.scala @@ -28,6 +28,7 @@ import io.viash.helpers.Bash @description("""An executable Scala script. |When defined in functionality.resources, only the first entry will be executed when running the built component or when running `viash run`. |When defined in functionality.test_resources, all entries will be executed during `viash test`.""".stripMargin) +@subclass("scala_script") case class ScalaScript( path: Option[String] = None, text: Option[String] = None, @@ -128,18 +129,11 @@ case class ScalaScript( ScriptInjectionMods(params = paramsCode.mkString) } - - def command(script: String): String = { - "scala -nc \"" + script + "\"" - } - - def commandSeq(script: String): Seq[String] = { - Seq("scala", "-nc", script) - } } object ScalaScript extends ScriptCompanion { val commentStr = "//" val extension = "scala" val `type` = "scala_script" + val executor = Seq("scala", "-nc") } \ No newline at end of file diff --git a/src/main/scala/io/viash/functionality/resources/Script.scala b/src/main/scala/io/viash/functionality/resources/Script.scala index 9eea16eb4..ceeb50625 100644 --- a/src/main/scala/io/viash/functionality/resources/Script.scala +++ b/src/main/scala/io/viash/functionality/resources/Script.scala @@ -69,14 +69,15 @@ trait Script extends Resource { }) } - def command(script: String): String - def commandSeq(script: String): Seq[String] + def command(script: String): String = (companion.executor :+ s"\"$script\"").mkString(" ") + def commandSeq(script: String): Seq[String] = companion.executor ++ Seq(script) } trait ScriptCompanion { val commentStr: String val extension: String val `type`: String + val executor: Seq[String] // def apply( // path: Option[String] = None, // text: Option[String] = None, diff --git a/src/main/scala/io/viash/functionality/resources/package.scala b/src/main/scala/io/viash/functionality/resources/package.scala index 948371310..67292e1ad 100644 --- a/src/main/scala/io/viash/functionality/resources/package.scala +++ b/src/main/scala/io/viash/functionality/resources/package.scala @@ -26,6 +26,7 @@ import io.circe.ACursor package object resources { + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ import io.viash.helpers.circe._ implicit val encodeURI: Encoder[URI] = Encoder.instance { @@ -79,15 +80,15 @@ package object resources { } })} - implicit val decodeBashScript: Decoder[BashScript] = deriveConfiguredDecoder[BashScript].prepare { setDestToPathOrDefault("./script.sh") } - implicit val decodePythonScript: Decoder[PythonScript] = deriveConfiguredDecoder[PythonScript].prepare { setDestToPathOrDefault("./script.py") } - implicit val decodeRScript: Decoder[RScript] = deriveConfiguredDecoder[RScript].prepare { setDestToPathOrDefault("./script.R") } - implicit val decodeJavaScriptScript: Decoder[JavaScriptScript] = deriveConfiguredDecoder[JavaScriptScript].prepare { setDestToPathOrDefault("./script.js") } - implicit val decodeNextflowScript: Decoder[NextflowScript] = deriveConfiguredDecoder[NextflowScript].prepare { setDestToPathOrDefault("./script.nf") } - implicit val decodeScalaScript: Decoder[ScalaScript] = deriveConfiguredDecoder[ScalaScript].prepare { setDestToPathOrDefault("./script.scala") } - implicit val decodeCSharpScript: Decoder[CSharpScript] = deriveConfiguredDecoder[CSharpScript].prepare { setDestToPathOrDefault("./script.csx") } - implicit val decodeExecutable: Decoder[Executable] = deriveConfiguredDecoder - implicit val decodePlainFile: Decoder[PlainFile] = deriveConfiguredDecoder[PlainFile].prepare { setDestToPathOrDefault("./text.txt") } + implicit val decodeBashScript: Decoder[BashScript] = deriveConfiguredDecoderFullChecks[BashScript].prepare { setDestToPathOrDefault("./script.sh") } + implicit val decodePythonScript: Decoder[PythonScript] = deriveConfiguredDecoderFullChecks[PythonScript].prepare { setDestToPathOrDefault("./script.py") } + implicit val decodeRScript: Decoder[RScript] = deriveConfiguredDecoderFullChecks[RScript].prepare { setDestToPathOrDefault("./script.R") } + implicit val decodeJavaScriptScript: Decoder[JavaScriptScript] = deriveConfiguredDecoderFullChecks[JavaScriptScript].prepare { setDestToPathOrDefault("./script.js") } + implicit val decodeNextflowScript: Decoder[NextflowScript] = deriveConfiguredDecoderFullChecks[NextflowScript].prepare { setDestToPathOrDefault("./script.nf") } + implicit val decodeScalaScript: Decoder[ScalaScript] = deriveConfiguredDecoderFullChecks[ScalaScript].prepare { setDestToPathOrDefault("./script.scala") } + implicit val decodeCSharpScript: Decoder[CSharpScript] = deriveConfiguredDecoderFullChecks[CSharpScript].prepare { setDestToPathOrDefault("./script.csx") } + implicit val decodeExecutable: Decoder[Executable] = deriveConfiguredDecoderFullChecks + implicit val decodePlainFile: Decoder[PlainFile] = deriveConfiguredDecoderFullChecks[PlainFile].prepare { setDestToPathOrDefault("./text.txt") } implicit def decodeResource: Decoder[Resource] = Decoder.instance { cursor => @@ -102,11 +103,8 @@ package object resources { case Right("csharp_script") => decodeCSharpScript.widen case Right("executable") => decodeExecutable.widen case Right("file") => decodePlainFile.widen - case Right(typ) => throw new RuntimeException( - "File type " + typ + " is not recognised. Should be one of " + - Script.companions.mkString("'", "', '", "'") + - ", or 'file'." - ) + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[BashScript](typ, Script.companions.map(c => c.`type`) ++ List("executable", "file")).widen case Left(_) => decodePlainFile.widen // default is a simple file } diff --git a/src/main/scala/io/viash/helpers/IO.scala b/src/main/scala/io/viash/helpers/IO.scala index d5a48237d..1c737e17f 100644 --- a/src/main/scala/io/viash/helpers/IO.scala +++ b/src/main/scala/io/viash/helpers/IO.scala @@ -32,11 +32,14 @@ import java.nio.file.attribute.PosixFilePermission import java.util.Comparator import java.io.FileNotFoundException import scala.jdk.CollectionConverters._ +import java.nio.charset.MalformedInputException +import io.viash.exceptions.{MalformedInputException => ViashMalformedInputException} +import io.viash.helpers.Logging /** * IO helper object for handling various file and directory operations. */ -object IO { +object IO extends Logging { /** * Returns the temporary directory path. @@ -107,6 +110,9 @@ object IO { } try { txtSource.getLines().mkString("\n") + } catch { + case e: MalformedInputException => throw new ViashMalformedInputException(uri.toString(), e) + case e: Throwable => throw e } finally { txtSource.close() } @@ -123,7 +129,7 @@ object IO { Some(read(uri)) } catch { case _: Exception => - Console.err.println(s"File at URI '$uri' not found") + info(s"File at URI '$uri' not found") None } } diff --git a/src/main/scala/io/viash/helpers/Logger.scala b/src/main/scala/io/viash/helpers/Logger.scala new file mode 100644 index 000000000..81d1c7f1a --- /dev/null +++ b/src/main/scala/io/viash/helpers/Logger.scala @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers + +import scala.io.AnsiColor + +object LoggerLevel extends Enumeration { + type Level = Value + val Error = Value(3) + val Warn = Value(4) + val Info = Value(6) + val Debug = Value(7) + val Trace = Value(8) + + val Success = Value(5) // Should have been 6, same as Info. Luckely we had a spare spot between 4 and 6 + + def fromString(level: String): Value = { + level match { + case "error" => Error + case "warn" => Warn + case "info" => Info + case "debug" => Debug + case "trace" => Trace + case _ => throw new IllegalArgumentException(level) + } + } +} + +object LoggerOutput extends Enumeration { + type Output = Value + val StdOut = Value(1) + val StdErr = Value(2) +} + +/** Partial logging facade styled alike SLF4J. + * Used Grizzled slf4j as further inspiration and basis. + */ +class Logger(val name: String, val level: LoggerLevel.Level, val useColor: Boolean) { + import LoggerOutput._ + import LoggerLevel._ + + @inline final def isErrorEnabled = isEnabled(Error) + @inline final def isWarnEnabled = isEnabled(Warn) + @inline final def isInfoEnabled = isEnabled(Info) + @inline final def isDebugEnabled = isEnabled(Debug) + @inline final def isTraceEnabled = isEnabled(Trace) + + @inline final def error(msg: => Any): Unit = _log(Error, msg) + @inline final def warn(msg: => Any): Unit = _log(Warn, msg) + @inline final def info(msg: => Any): Unit = _log(Info, msg) + @inline final def debug(msg: => Any): Unit = _log(Debug, msg) + @inline final def trace(msg: => Any): Unit = _log(Trace, msg) + @inline final def success(msg: => Any): Unit = _log(Success, msg) + + @inline final def errorOut(msg: => Any): Unit = _logOut(Error, msg) + @inline final def warnOut(msg: => Any): Unit = _logOut(Warn, msg) + @inline final def infoOut(msg: => Any): Unit = _logOut(Info, msg) + @inline final def debugOut(msg: => Any): Unit = _logOut(Debug, msg) + @inline final def traceOut(msg: => Any): Unit = _logOut(Trace, msg) + @inline final def successOut(msg: => Any): Unit = _logOut(Success, msg) + + @inline final def isEnabled(level: Level): Boolean = this.level >= level + + @inline private def _colorString(level: Level): String = + level match { + case Error => AnsiColor.RED + case Warn => AnsiColor.YELLOW + case Info => AnsiColor.WHITE + case Debug => AnsiColor.GREEN + case Trace => AnsiColor.CYAN + + case Success => AnsiColor.GREEN + } + + @inline private def _log(level: Level, msg: => Any): Unit = { + if (!isEnabled(level)) return + + if (useColor) + Console.err.println(s"${_colorString(level)}${msg.toString()}${AnsiColor.RESET}") + else + Console.err.println(msg.toString()) + } + + @inline private def _logOut(level: Level, msg: => Any): Unit = { + if (!isEnabled(level)) return + + if (useColor) + Console.out.println(s"${_colorString(level)}${msg.toString()}${AnsiColor.RESET}") + else + Console.out.println(msg.toString()) + } + + @inline def log(out: Output, level: Level, color: String, msg: => Any): Unit = { + if (!isEnabled(level)) return + + val printer = + if (out == LoggerOutput.StdErr) + Console.err + else + Console.out + + if (useColor) + printer.println(s"${color}${msg.toString()}${AnsiColor.RESET}") + else + printer.println(msg.toString()) + } +} + +trait Logging { + // The logger. Instantiated the first time it's used. + @transient private lazy val _logger = Logger(getClass) + + protected def logger: Logger = _logger + protected def loggerName = logger.name + + protected def isErrorEnabled = logger.isErrorEnabled + protected def isWarnEnabled = logger.isWarnEnabled + protected def isInfoEnabled = logger.isInfoEnabled + protected def isDebugEnabled = logger.isDebugEnabled + protected def isTraceEnabled = logger.isTraceEnabled + + protected def error(msg: => Any): Unit = logger.error(msg) + protected def warn(msg: => Any): Unit = logger.warn(msg) + protected def info(msg: => Any): Unit = logger.info(msg) + protected def debug(msg: => Any): Unit = logger.debug(msg) + protected def trace(msg: => Any): Unit = logger.trace(msg) + protected def success(msg: => Any): Unit = logger.success(msg) + + protected def errorOut(msg: => Any): Unit = logger.errorOut(msg) + protected def warnOut(msg: => Any): Unit = logger.warnOut(msg) + protected def infoOut(msg: => Any): Unit = logger.infoOut(msg) + protected def debugOut(msg: => Any): Unit = logger.debugOut(msg) + protected def traceOut(msg: => Any): Unit = logger.traceOut(msg) + protected def successOut(msg: => Any): Unit = logger.successOut(msg) + + protected def log(out: LoggerOutput.Output, level: LoggerLevel.Level, color: String, msg: => Any): Unit = logger.log(out, level, color, msg) +} + +object Logger { + import scala.reflect.{classTag, ClassTag} + + val rootLoggerName = "Viash-root-logger" + + def apply(name: String, level: LoggerLevel.Level, useColor: Boolean): Logger = new Logger(name, level, useColor) + def apply(name: String): Logger = new Logger(name, getLoggerLevel(name), useColor) + def apply(cls: Class[_]): Logger = apply(cls.getName) + def apply[C: ClassTag](): Logger = apply(classTag[C].runtimeClass.getName) + + def rootLogger = apply(rootLoggerName) + + object UseLevelOverride extends util.DynamicVariable[LoggerLevel.Level](LoggerLevel.Info) + def getLoggerLevel(name: String): LoggerLevel.Level = { + if (name != rootLoggerName) // prevent constructor loop + rootLogger.debug(s"GetLoggerLevel for $name") + + if (name == "io.viash.helpers.LoggerTest$ClassTraitLoggingTest$1") + return LoggerLevel.Trace + + // TODO setting of logger levels for individual loggers + UseLevelOverride.value + } + + object UseColorOverride extends util.DynamicVariable[Option[Boolean]](None) + private val useColor_ = System.console() != null + def useColor = UseColorOverride.value.getOrElse(useColor_) +} diff --git a/src/main/scala/io/viash/helpers/SysEnv.scala b/src/main/scala/io/viash/helpers/SysEnv.scala new file mode 100644 index 000000000..6e30349c0 --- /dev/null +++ b/src/main/scala/io/viash/helpers/SysEnv.scala @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers + +import io.viash.schemas._ + +@nameOverride("EnvironmentVariables") +@description("Viash checks several environment variables during operation.") +@documentFully +trait SysEnvTrait { + @nameOverride("VIASH_HOME") + @description( + """If `VIASH_HOME` is not defined, the fallback `HOME`/.viash is used. + | + |Location where specific downloaded versions of Viash will be cached and run from. + |""".stripMargin) + def viashHome: String + + @nameOverride("VIASH_VERSION") + @description( + """A specific Viash version can be set to run the commands with. If so required, the specific Viash version will be downloaded. + |This is useful when replicating older results or building Viash components that use outdated code. + |""".stripMargin) + @example( + """VIASH_VERSION=0.7.0 viash ns build""", + "sh" + ) + def viashVersion: Option[String] +} + + +object SysEnv extends SysEnvTrait { + private val sysEnvOverride = scala.collection.mutable.Map.empty[String, String] + + private def get(key: String): Option[String] = sysEnvOverride.get(key) orElse sys.env.get(key) + private def getOrElse(key: String, default: => String): String = get(key) match { + case Some(v) => v + case None => default + } + + private lazy val codeRunInTestBench = { getClass.getPackage.getImplementationTitle == null } + + def set(key: String, value: String) = { + if (!codeRunInTestBench) + throw new IllegalAccessException("SysEnv.set is not allowed in production code.") + sysEnvOverride.addOne(key -> value) + } + def remove(key: String) = { + if (!codeRunInTestBench) + throw new IllegalAccessException("SysEnv.remove is not allowed in production code.") + sysEnvOverride.remove(key) + } + + def viashHome = getOrElse("VIASH_HOME", sys.env("HOME") + "/.viash") + def viashVersion = get("VIASH_VERSION") +} diff --git a/src/main/scala/io/viash/helpers/Yaml.scala b/src/main/scala/io/viash/helpers/Yaml.scala new file mode 100644 index 000000000..1760a8a76 --- /dev/null +++ b/src/main/scala/io/viash/helpers/Yaml.scala @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers + +import org.yaml.snakeyaml.nodes._ +import org.yaml.snakeyaml.{Yaml => Syaml} +import java.io.{StringReader, StringWriter} +import scala.util.Using + +object Yaml { + /** + * Change valid number +.inf, -.inf, or .nan values and change them to string values + * + * Json does not support these values so we must pass them as strings. + * + * @param data Yaml data as text + * @return Yaml data as text + */ + def replaceInfinities(data: String): String = { + Using.Manager { use => + val yaml = new Syaml() + + // Convert yaml text to Node tree + val yamlTree = yaml.compose(new StringReader(data)) + + // Search for number values of "+.inf" and replace them in place + recurseReplaceInfinities(yamlTree) + + // Save yaml back to string + val writer = new StringWriter() + yaml.serialize(yamlTree, writer) + writer.toString + }.get + } + + private def recurseReplaceInfinities(node: Node): Unit = node match { + // Traverse maps + case mapNode: MappingNode => mapNode.getValue().forEach(t => recurseReplaceInfinities(t.getValueNode())) + // Traverse arrays + case seqNode: SequenceNode => seqNode.getValue().forEach(n => recurseReplaceInfinities(n)) + // If double and string matches, change type from float to string. + // The value can stay the same and instead will get escaped during serialization. + case scalar: ScalarNode if scalar.getTag == Tag.FLOAT => + if ("([-+]?\\.(inf|Inf|INF))|\\.(nan|NaN|NAN)".r matches scalar.getValue()) + scalar.setTag(Tag.STR) + // No changes required + case _ => + } +} diff --git a/src/main/scala/io/viash/helpers/circe/Convert.scala b/src/main/scala/io/viash/helpers/circe/Convert.scala new file mode 100644 index 000000000..b9e1d18e1 --- /dev/null +++ b/src/main/scala/io/viash/helpers/circe/Convert.scala @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers.circe + +import io.circe.Json +import io.circe.yaml.parser +import scala.util.{Try, Success, Failure} +import io.viash.exceptions.{ConfigYamlException, ConfigParserException} +import io.viash.helpers.circe._ +import io.viash.schemas._ +import io.circe.Decoder + +object Convert { + + private def parsingYamlErrorHandler[C](pathStr: String)(e: Exception): C = { + // Console.err.println(s"${Console.RED}Error parsing, invalid Yaml structure '${uri}'.${Console.RESET}\nDetails:") + // throw e + throw new ConfigYamlException(pathStr, e) + } + private def parsingErrorHandler[C](pathStr: String)(e: Exception): C = { + // Console.err.println(s"${Console.RED}Error parsing '${uri}'.${Console.RESET}\nDetails:") + // throw e + throw new ConfigParserException(pathStr, e) + } + + def textToJson(text: String, pathStr: String) = { + parser.parse(text).fold(parsingYamlErrorHandler(pathStr), identity) + } + + def jsonToClass[T](json: Json, pathStr: String)(implicit decoder: Decoder[T]): T = { + Try(json.as[T]) match { + case Success(res) => res.fold(parsingErrorHandler(pathStr), identity) + case Failure(e) => throw new ConfigParserException(pathStr, e) + } + } + +} diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala new file mode 100644 index 000000000..cab16076c --- /dev/null +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderFullChecks.scala @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers.circe + +import shapeless.Lazy +import scala.reflect.runtime.universe._ + +import io.circe.Decoder +import io.circe.generic.extras.decoding.ConfiguredDecoder +import io.circe.generic.extras.semiauto.deriveConfiguredDecoder + +object DeriveConfiguredDecoderFullChecks { + import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderWithValidationCheck._ + + def deriveConfiguredDecoderFullChecks[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + .validate( + validator[A], + s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." + ) + .prepare( DeriveConfiguredDecoderWithDeprecationCheck.checkDeprecation[A] ) +} diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala index 8d437212c..da5e58157 100644 --- a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithDeprecationCheck.scala @@ -27,7 +27,9 @@ import shapeless.Lazy import io.viash.schemas.ParameterSchema import io.circe.ACursor -object DeriveConfiguredDecoderWithDeprecationCheck { +import io.viash.helpers.Logging + +object DeriveConfiguredDecoderWithDeprecationCheck extends Logging { private def memberDeprecationCheck(name: String, history: List[CursorOp], T: Type): Unit = { val m = T.member(TermName(name)) @@ -37,12 +39,12 @@ object DeriveConfiguredDecoderWithDeprecationCheck { if (deprecated.isDefined) { val d = deprecated.get val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - Console.err.println(s"Warning: .$historyString.$name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + info(s"Warning: .$historyString.$name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") } if (removed.isDefined) { val r = removed.get val historyString = history.collect{ case df: CursorOp.DownField => df.k }.reverse.mkString(".") - Console.err.println(s"Error: .$historyString.$name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") + info(s"Error: .$historyString.$name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") } } @@ -54,11 +56,11 @@ object DeriveConfiguredDecoderWithDeprecationCheck { val removed = schema.flatMap(_.removed) if (deprecated.isDefined) { val d = deprecated.get - Console.err.println(s"Warning: $name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") + info(s"Warning: $name is deprecated: ${d.message} Deprecated since ${d.deprecation}, planned removal ${d.removal}.") } if (removed.isDefined) { val r = removed.get - Console.err.println(s"Error: $name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}") + info(s"Error: $name was removed: ${r.message} Initially deprecated ${r.deprecation}, removed ${r.removal}.") } } diff --git a/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala new file mode 100644 index 000000000..1b1baf8b4 --- /dev/null +++ b/src/main/scala/io/viash/helpers/circe/DeriveConfiguredDecoderWithValidationCheck.scala @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.helpers.circe + +import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +import io.circe.{ Decoder, CursorOp } +import io.circe.generic.extras.decoding.ConfiguredDecoder + +import scala.reflect.runtime.universe._ +import shapeless.Lazy + +import io.viash.schemas.ParameterSchema +import io.circe.ACursor +import io.viash.exceptions.ConfigParserSubTypeException +import io.viash.exceptions.ConfigParserValidationException +import io.circe.HCursor + +object DeriveConfiguredDecoderWithValidationCheck { + + // Validate the json can correctly converted to the required type by actually converting it. + // Throw an exception when the conversion fails. + def validator[A](pred: HCursor)(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Boolean = { + val d = deriveConfiguredDecoder[A] + val v = d(pred) + + v.fold(_ => { + throw new ConfigParserValidationException(typeOf[A].baseClasses.head.fullName, pred.value.toString()) + false + }, _ => true) + } + + // Attempts to convert the json to the desired class. Throw an exception if the conversion fails. + def deriveConfiguredDecoderWithValidationCheck[A](implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + .validate( + validator[A], + s"Could not convert json to ${typeOf[A].baseClasses.head.fullName}." + ) + + // Dummy decoder to generate exceptions when an invalid type is specified + // We need a valid class type to be specified + def invalidSubTypeDecoder[A](tpe: String, validTypes: List[String])(implicit decode: Lazy[ConfiguredDecoder[A]], tag: TypeTag[A]): Decoder[A] = deriveConfiguredDecoder[A] + .validate( + pred => { + throw new ConfigParserSubTypeException(tpe, validTypes, pred.value.toString()) + false + }, + s"Type '$tpe 'is not recognised. Valid types are ${validTypes.mkString("'", "', '", ",")}." + ) + +} diff --git a/src/main/scala/io/viash/helpers/circe/RichJson.scala b/src/main/scala/io/viash/helpers/circe/RichJson.scala index 18b4178c1..01f31b991 100644 --- a/src/main/scala/io/viash/helpers/circe/RichJson.scala +++ b/src/main/scala/io/viash/helpers/circe/RichJson.scala @@ -18,9 +18,13 @@ package io.viash.helpers.circe import java.net.URI + import io.circe.Json import io.circe.JsonObject import io.circe.generic.extras.Configuration +import io.circe.{Json, Printer => JsonPrinter} +import io.circe.yaml.{Printer => YamlPrinter} + import io.viash.helpers.IO class RichJson(json: Json) { @@ -231,4 +235,27 @@ class RichJson(json: Json) { _.map(_.stripInherits) ) } + + private val yamlPrinter = YamlPrinter( + preserveOrder = true, + dropNullKeys = true, + mappingStyle = YamlPrinter.FlowStyle.Block, + splitLines = true, + stringStyle = YamlPrinter.StringStyle.DoubleQuoted + ) + private val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) + + /** + * Convert to a pretty String. + * + * @param format Must be either 'yaml' or 'json' + * @return The YAML or JSON String representation. + */ + def toFormattedString(format: String): String = { + format match { + case "yaml" => yamlPrinter.pretty(json) + case "json" => jsonPrinter.print(json) + case _ => throw new IllegalArgumentException("'format' must be either 'json' or 'yaml'.") + } + } } \ No newline at end of file diff --git a/src/main/scala/io/viash/platforms/DockerPlatform.scala b/src/main/scala/io/viash/platforms/DockerPlatform.scala index 4e0974c8a..784402e82 100644 --- a/src/main/scala/io/viash/platforms/DockerPlatform.scala +++ b/src/main/scala/io/viash/platforms/DockerPlatform.scala @@ -46,9 +46,11 @@ import io.viash.helpers.Escaper | packages: [ curl ] |""".stripMargin, "yaml") +@subclass("docker") case class DockerPlatform( @description("As with all platforms, you can give a platform a different name. By specifying `id: foo`, you can target this platform (only) by specifying `-p foo` in any of the Viash commands.") @example("id: foo", "yaml") + @default("docker") id: String = "docker", @description("The base container to start from. You can also add the tag here if you wish.") @@ -84,13 +86,16 @@ case class DockerPlatform( @description("The separator between the namespace and the name of the component, used for determining the image name. Default: `\"/\"`.") @example("namespace_separator: \"_\"", "yaml") + @default("/") namespace_separator: String = "/", @description("Enables or disables automatic volume mapping. Enabled when set to `Automatic` or disabled when set to `Manual`. Default: `Automatic`.") + @default("Automatic") resolve_volume: DockerResolveVolume = Automatic, @description("In Linux, files created by a Docker container will be owned by `root`. With `chown: true`, Viash will automatically change the ownership of output files (arguments with `type: file` and `direction: output`) to the user running the Viash command after execution of the component. Default value: `true`.") @example("chown: false", "yaml") + @default("True") chown: Boolean = true, @description("A list of enabled ports. This doesn't change the Dockerfile but gets added as a command-line argument at runtime.") @@ -100,6 +105,7 @@ case class DockerPlatform( | - 8080 |""".stripMargin, "yaml") + @default("Empty") port: OneOrMore[String] = Nil, @description("The working directory when starting the container. This doesn't change the Dockerfile but gets added as a command-line argument at runtime.") @@ -127,9 +133,11 @@ case class DockerPlatform( + +""".stripMargin('+')) @example("setup_strategy: alwaysbuild", "yaml") + @default("ifneedbepullelsecachedbuild") setup_strategy: DockerSetupStrategy = IfNeedBePullElseCachedBuild, @description("Add [docker run](https://docs.docker.com/engine/reference/run/) arguments.") + @default("Empty") run_args: OneOrMore[String] = Nil, @description("The source of the target image. This is used for defining labels in the dockerfile.") @@ -152,10 +160,12 @@ case class DockerPlatform( | |The order in which these dependencies are specified determines the order in which they will be installed. |""".stripMargin) + @default("Empty") setup: List[Requirements] = Nil, @description("Additional requirements specific for running unit tests.") @since("Viash 0.5.13") + @default("Empty") test_setup: List[Requirements] = Nil, @description("Override the entrypoint of the base container. Default set `ENTRYPOINT []`.") @@ -163,6 +173,7 @@ case class DockerPlatform( @exampleWithDescription("""entrypoint: ["top", "-b"]""", "yaml", "Entrypoint of the container in the exec format, which is the prefered form.") @exampleWithDescription("""entrypoint: "top -b"""", "yaml", "Entrypoint of the container in the shell format.") @since("Viash 0.7.4") + @default("[]") entrypoint: Option[Either[String, List[String]]] = Some(Right(Nil)), @description("Set the default command being executed when running the Docker container.") @@ -172,92 +183,6 @@ case class DockerPlatform( cmd: Option[Either[String, List[String]]] = None ) extends Platform { - // START OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - @description("Adds a `privileged` flag to the docker run.") - @removed("Add a `privileged` flag in `run_args` instead, e.g. `{type: docker, run_args: \"--privileged\"}`.", "0.6.3", "0.7.0") - private val privileged: Option[Boolean] = None - - @description("Specify which apk packages should be available in order to run the component.") - @example( - """setup: - | - type: apk - | packages: [ sl ] - |""".stripMargin, - "yaml") - @removed("Use `setup` instead, e.g. `{type: docker, setup: [{ type: apk, ... }]}`. Will be removed.", "0.5.15", "0.7.0") - private val apk: Option[ApkRequirements] = None - - @description("Specify which apt packages should be available in order to run the component.") - @example( - """setup: - | - type: apt - | packages: [ sl ] - |""".stripMargin, - "yaml") - @removed("Use `setup` instead, e.g. `{type: docker, setup: [{ type: apt, ... }]}`. Will be removed.", "0.5.15", "0.7.0") - private val apt: Option[AptRequirements] = None - - @description("Specify which yum packages should be available in order to run the component.") - @example( - """setup: - | - type: yum - | packages: [ sl ] - |""".stripMargin, - "yaml") - @removed("Use `setup` instead, e.g. `{type: docker, setup: [{ type: yum, ... }]}`. Will be removed.", "0.5.15", "0.7.0") - private val yum: Option[YumRequirements] = None - - @description("Specify which R packages should be available in order to run the component.") - @example( - """setup: - | - type: r - | cran: [ dynutils ] - | bioc: [ AnnotationDbi ] - | git: [ https://some.git.repository/org/repo ] - | github: [ rcannood/SCORPIUS ] - | gitlab: [ org/package ] - | svn: [ https://path.to.svn/group/repo ] - | url: [ https://github.com/hadley/stringr/archive/HEAD.zip ] - | script: [ 'devtools::install(".")' ] - |""".stripMargin, - "yaml") - @removed("Use `setup` instead, e.g. `{type: docker, setup: [{ type: r, ... }]}`. Will be removed.", "0.5.15", "0.7.0") - private val r: Option[RRequirements] = None - - @description("Specify which Python packages should be available in order to run the component.") - @example( - """setup: - | - type: python - | pip: [ numpy ] - | git: [ https://some.git.repository/org/repo ] - | github: [ jkbr/httpie ] - | gitlab: [ foo/bar ] - | mercurial: [ http://... ] - | svn: [ http://...] - | bazaar: [ http://... ] - | url: [ http://... ] - |""".stripMargin, - "yaml") - @removed("Use `setup` instead, e.g. `{type: docker, setup: [{ type: python, ... }]}`. Will be removed.", "0.5.15", "0.7.0") - private val python: Option[PythonRequirements] = None - - @description("Specify which Docker commands should be run during setup.") - @example( - """setup: - | - type: docker - | build_args: [ GITHUB_PAT=hello_world ] - | run: [ git clone ... ] - | add: [ "http://foo.bar ." ] - | copy: [ "http://foo.bar ." ] - | resources: - | - resource.txt /path/to/resource.txt - |""".stripMargin, - "yaml") - @removed("Use `setup` instead, e.g. `{type: docker, setup: [{ type: docker, ... }]}`. Will be removed.", "0.5.15", "0.7.0") - private val docker: Option[DockerRequirements] = None - - // END OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - @internalFunctionality override val hasSetup = true @@ -561,7 +486,7 @@ case class DockerPlatform( | IFS='${Bash.escapeString(arg.multiple_sep, quote = true)}' | for var in $$${arg.VIASH_PAR}; do | unset IFS - | VIASH_EXTRA_MOUNTS+=( $$(ViashAutodetectMountArg "$$var") ) + | VIASH_EXTRA_MOUNTS+=( "$$(ViashAutodetectMountArg "$$var")" ) | var=$$(ViashAutodetectMount "$$var") | $viash_temp+=( "$$var" )$chownIfOutput | done @@ -572,7 +497,7 @@ case class DockerPlatform( Some( s""" |if [ ! -z "$$${arg.VIASH_PAR}" ]; then - | VIASH_EXTRA_MOUNTS+=( $$(ViashAutodetectMountArg "$$${arg.VIASH_PAR}") ) + | VIASH_EXTRA_MOUNTS+=( "$$(ViashAutodetectMountArg "$$${arg.VIASH_PAR}")" ) | ${arg.VIASH_PAR}=$$(ViashAutodetectMount "$$${arg.VIASH_PAR}")$chownIfOutput |fi""".stripMargin) case _ => None diff --git a/src/main/scala/io/viash/platforms/NativePlatform.scala b/src/main/scala/io/viash/platforms/NativePlatform.scala index f4727cffe..5e27e36b0 100644 --- a/src/main/scala/io/viash/platforms/NativePlatform.scala +++ b/src/main/scala/io/viash/platforms/NativePlatform.scala @@ -33,9 +33,11 @@ import io.viash.schemas._ | - type: native |""".stripMargin, "yaml") +@subclass("native") case class NativePlatform( @description("As with all platforms, you can give a platform a different name. By specifying `id: foo`, you can target this platform (only) by specifying `-p foo` in any of the Viash commands.") @example("id: foo", "yaml") + @default("native") id: String = "native", `type`: String = "native" ) extends Platform { diff --git a/src/main/scala/io/viash/platforms/NextflowLegacyPlatform.scala b/src/main/scala/io/viash/platforms/NextflowLegacyPlatform.scala deleted file mode 100644 index 4a79507d2..000000000 --- a/src/main/scala/io/viash/platforms/NextflowLegacyPlatform.scala +++ /dev/null @@ -1,900 +0,0 @@ -/* - * Copyright (C) 2020 Data Intuitive - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package io.viash.platforms - -import io.viash.config.Config -import io.viash.functionality._ -import io.viash.functionality.resources._ -import io.viash.functionality.arguments._ -import io.viash.helpers.{Docker, Bash} -import io.viash.helpers.data_structures._ -import io.viash.schemas._ -import io.viash.helpers.Escaper - -/** - * / * Platform class for generating NextFlow (DSL2) modules. - */ -@description("Run a Viash component as a Nextflow module.") -@removed("Nextflow platform with `variant: legacy` was removed", "0.6.0", "0.7.0") -case class NextflowLegacyPlatform( - @description("Every platform can be given a specific id that can later be referred to explicitly when running or building the Viash component.") - id: String = "nextflow", - - @description( - """If no image attributes are configured, Viash will use the auto-generated image name from the Docker platform: - | - |``` - |[/]: - |``` - |It's possible to specify the container image explicitly with which to run the module in different ways: - | - |``` - |image: dataintuitive/viash:0.4.0 - |``` - |Exactly the same can be obtained with - | - |``` - |image: dataintuitive/viash - |registry: index.docker.io/v1/ - |tag: 0.4.0 - |``` - |Specifying the attribute(s) like this will use the container `dataintuitive/viash:0.4.0` from Docker hub (registry). - | - |If no tag is specified Viash will use `functionality.version` as the tag. - | - |If no registry is specified, Viash (and NextFlow) will assume the image is available locally or on Docker Hub. In other words, the `registry: ...` attribute above is superfluous. No other registry is checked automatically due to a limitation from Docker itself. - |""".stripMargin) - image: Option[String], - - @description("Specify a Docker image based on its tag.") - @example("tag: 4.0", "yaml") - tag: Option[String] = None, - - @description("The URL to the a [custom Docker registry](https://docs.docker.com/registry/).") - @example("registry: https://my-docker-registry.org", "yaml") - registry: Option[String] = None, - - @description("Name of a container's [organization](https://docs.docker.com/docker-hub/orgs/).") - @example("organization: viash-io", "yaml") - organization: Option[String] = None, - - @description("The default namespace separator is \"_\".") - @example("namespace_separator: \"+\"", "yaml") - namespace_separator: String = "_", - - @description( - """NextFlow uses the autogenerated `work` dirs to manage process IO under the hood. In order effectively output something one can publish the results a module or step in the pipeline. In order to do this, add `publish: true` to the config: - | - | - publish is optional - | - Default value is false - | - |This attribute simply defines if output of a component should be published yes or no. The output location has to be provided at pipeline launch by means of the option `--publishDir ...` or as `params.publishDir` in `nextflow.config`: - |``` - |params.publishDir = "..." - |``` - |""".stripMargin) - publish: Option[Boolean] = None, - - @description( - """By default, a subdirectory is created corresponding to the unique ID that is passed in the triplet. Let us illustrate this with an example. The following code snippet uses the value of `--input` as an input of a workflow. The input can include a wildcard so that multiple samples can run in parallel. We use the parent directory name (`.getParent().baseName`) as an identifier for the sample. We pass this as the first entry of the triplet: - | - |``` - |Channel.fromPath(params.input) \ - | | map{ it -> [ it.getParent().baseName , it ] } \ - | | map{ it -> [ it[0] , it[1], params ] } - | | ... - |``` - |Say the resulting sample names are `SAMPLE1` and `SAMPLE2`. The next step in the pipeline will be published (at least by default) under: - |``` - |/SAMPLE1/ - |/SAMPLE2/ - |``` - |These per-ID subdirectories can be avoided by setting: - |``` - |per_id: false - |``` - |""".stripMargin) - per_id: Option[Boolean] = None, - - @description("Separates the outputs generated by a Nextflow component with multiple outputs as separate events on the channel. Default value: `true`.") - @example("separate_multiple_outputs: false", "yaml") - separate_multiple_outputs: Boolean = true, - - @description( - """When `publish: true`, this attribute defines where the output is written relative to the `params.publishDir` setting. For example, `path: processed` in combination with `--output s3://some_bucket/` will store the output of this component under - |``` - |s3://some_bucket/processed/ - |``` - |This attribute gives control over the directory structure of the output. For example: - |``` - |path: raw_data - |``` - |Or even: - |``` - |path: raw_data/bcl - |``` - |Please note that `per_id` and `path` can be combined. - |""".stripMargin) - path: Option[String] = None, - - @description( - """When running the module in a cluster context and depending on the cluster type, [NextFlow allows for attaching labels](https://www.nextflow.io/docs/latest/process.html#label) to the process that can later be used as selectors for associating resources to this process. - | - |In order to attach one label to a process/component, one can use the `label: ...` attribute, multiple labels can be added using `labels: [ ..., ... ]` and the two can even be mixed. - | - |In the main `nextflow.config`, one can now use this label: - | - |process { - | ... - | withLabel: bigmem { - | maxForks = 5 - | ... - | } - |} - |""".stripMargin) - @example("label: highmem labels: [ highmem, highcpu ]", "yaml") - label: Option[String] = None, - - @description( - """When running the module in a cluster context and depending on the cluster type, [NextFlow allows for attaching labels](https://www.nextflow.io/docs/latest/process.html#label) to the process that can later be used as selectors for associating resources to this process. - | - |In order to attach one label to a process/component, one can use the `label: ...` attribute, multiple labels can be added using `labels: [ ..., ... ]` and the two can even be mixed. - | - |In the main `nextflow.config`, one can now use this label: - | - |process { - | ... - | withLabel: bigmem { - | maxForks = 5 - | ... - | } - |} - |""".stripMargin) - @example("label: highmem labels: [ highmem, highcpu ]", "yaml") - labels: OneOrMore[String] = Nil, - - @description( - """By default NextFlow will create a symbolic link to the inputs for a process/module and run the tool at hand using those symbolic links. Some applications do not cope well with this strategy, in that case the files should effectively be copied rather than linked to. This can be achieved by using `stageInMode: copy`. - |This attribute is optional, the default is `symlink`. - |""".stripMargin) - @example("stageInMode: copy", "yaml") - stageInMode: Option[String] = None, - - @undocumented - directive_cpus: Option[Integer] = None, - @undocumented - directive_max_forks: Option[Integer] = None, - @undocumented - directive_time: Option[String] = None, - @undocumented - directive_memory: Option[String] = None, - @undocumented - directive_cache: Option[String] = None, - `type`: String = "nextflow", - variant: String = "legacy" -) extends NextflowPlatform { - // START OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - @removed("Undocumented & stale value", "0.6.3", "0.7.0") - private val executor: Option[String] = None - @removed("nextflow platform: attribute 'version' was removed", "0.4.0", "0.7.0") - private val version: Option[String] = None - // END OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - - private val nativePlatform = NativePlatform(id = id) - - def modifyFunctionality(config: Config, testing: Boolean): Functionality = { - val functionality = config.functionality - import NextFlowUtils._ - implicit val fun: Functionality = functionality - - val fname = functionality.name - - // get image info - val imageInfo = Docker.getImageInfo( - functionality = Some(functionality), - registry = registry, - organization = organization, - name = image, - tag = tag.map(_.toString), - namespaceSeparator = namespace_separator - ) - - // get main script/binary - val mainResource = functionality.mainScript - val executionCode = mainResource match { - case Some(e: Executable) => e.path.get - case _ => fname - } - - val allPars = functionality.allArguments - - val outputs = allPars - .filter(_.isInstanceOf[FileArgument]) - .count(_.direction == Output) - - // All values for arguments/parameters are defined in the root of - // the params structure. the function name is prefixed as a namespace - // identifier. A "__" is used to separate namespace and arg/option. - // - // Required arguments also get a params. entry so that they can be - // called using --param value when using those standalone. - - /** - * A representation of viash's functionality.arguments is converted into a - * nextflow.config data structure under params..arguments. - * - * A `value` attribute is added that points to params.__. - * In turn, a pointer is configured to params.argument. - * - * The goal is to have a configuration file that works both when running - * the module standalone as well as in a pipeline. - */ - val namespacedParameters: List[ConfigTuple] = { - functionality.allArguments.flatMap { argument => (argument.required, argument.default.toList) match { - case (true, _) => - // println(s"> ${argument.plainName} in $fname is set to be required.") - // println(s"> --${argument.plainName} <...> has to be specified when running this module standalone.") - Some( - namespacedValueTuple( - argument.plainName.replace("-", "_"), - "viash_no_value" - )(fun) - ) - case (false, Nil) => - Some( - namespacedValueTuple( - argument.plainName.replace("-", "_"), - "no_default_value_configured" - )(fun) - ) - case (false, li) => - Some( - namespacedValueTuple( - argument.plainName.replace("-", "_"), - NextFlowUtils.escapeNextflowString(li.mkString(argument.multiple_sep.toString)) - )(fun) - ) - }} - } - - val argumentsAsTuple: List[ConfigTuple] = - if (functionality.allArguments.nonEmpty) { - List( - "arguments" -> NestedValue(functionality.allArguments.map(argumentToConfigTuple(_))) - ) - } else { - Nil - } - - val mainParams: List[ConfigTuple] = List( - "name" -> functionality.name, - "container" -> imageInfo.name, - "containerTag" -> imageInfo.tag, - "containerRegistry" -> imageInfo.registry.getOrElse(""), - "containerOrganization" -> imageInfo.organization.getOrElse(""), - "command" -> executionCode - ) - - // fetch test information - val tests = functionality.test_resources - val testPaths = tests.map(test => test.path.getOrElse("/dev/null")) - val testScript: List[String] = { - tests.flatMap{ - case test: Script => Some(test.filename) - case _ => None - } - } - - // If no tests are defined, isDefined is set to FALSE - val testConfig: List[ConfigTuple] = List("tests" -> NestedValue( - List( - tupleToConfigTuple("isDefined" -> tests.nonEmpty), - tupleToConfigTuple("testScript" -> testScript.headOption.getOrElse("NA")), - tupleToConfigTuple("testResources" -> testPaths) - ) - )) - - /** - * A few notes: - * 1. input and output are initialized as empty strings, so that no warnings appear. - * 2. id is initialized as empty string, which makes sense in test scenarios. - */ - val asNestedTuples: List[ConfigTuple] = List( - "docker.enabled" -> true, - "def viash_temp = System.getenv(\"VIASH_TEMP\") ?: \"/tmp/\"\n docker.runOptions" -> "-i -v ${baseDir}:${baseDir} -v $viash_temp:$viash_temp", - "process.container" -> "dataintuitive/viash", - "params" -> NestedValue( - namespacedParameters ::: - List( - tupleToConfigTuple("id" -> ""), - tupleToConfigTuple("testScript" -> testScript.headOption.getOrElse("")), // TODO: what about when there are multiple tests? - tupleToConfigTuple("testResources" -> testPaths), - tupleToConfigTuple(functionality.name -> NestedValue( - mainParams ::: - testConfig ::: - argumentsAsTuple - )) - ) - )) - - val setup_nextflowconfig = PlainFile( - dest = Some("nextflow.config"), - text = Some(listMapToConfig(asNestedTuples)) - ) - - val setup_main_header = - s"""nextflow.enable.dsl=2 - | - |params.test = false - |params.debug = false - |params.publishDir = "./" - |""".stripMargin - - val setup_main_outputFilters: String = { - if (separate_multiple_outputs) { - allPars - .filter(_.isInstanceOf[FileArgument]) - .filter(_.direction == Output) - .map(par => - s""" - |// A process that filters out ${par.plainName} from the output Map - |process filter${par.plainName.capitalize} { - | - | input: - | tuple val(id), val(input), val(_params) - | output: - | tuple val(id), val(output), val(_params) - | when: - | input.keySet().contains("${par.plainName}") - | exec: - | output = input["${par.plainName}"] - | - |}""".stripMargin - ).mkString("\n") - } else { - "" - } - } - - val setup_main_check = - s""" - |// A function to verify (at runtime) if all required arguments are effectively provided. - |def checkParams(_params) { - | _params.arguments.collect{ - | if (it.value == "viash_no_value") { - | println("[ERROR] option --$${it.name} not specified in component $fname") - | println("exiting now...") - | exit 1 - | } - | } - |} - | - |""".stripMargin - - - val setup_main_utils = - s""" - |def escape(str) { - | return str.replaceAll('\\\\\\\\', '\\\\\\\\\\\\\\\\').replaceAll("\\"", "\\\\\\\\\\"").replaceAll("\\n", "\\\\\\\\n").replaceAll("`", "\\\\\\\\`") - |} - | - |def renderArg(it) { - | if (it.flags == "") { - | return "\'" + escape(it.value) + "\'" - | } else if (it.type == "boolean_true") { - | if (it.value.toLowerCase() == "true") { - | return it.flags + it.name - | } else { - | return "" - | } - | } else if (it.type == "boolean_false") { - | if (it.value.toLowerCase() == "true") { - | return "" - | } else { - | return it.flags + it.name - | } - | } else if (it.value == "no_default_value_configured") { - | return "" - | } else { - | def retVal = it.value in List && it.multiple ? it.value.join(it.multiple_sep): it.value - | return it.flags + it.name + " \'" + escape(retVal) + "\'" - | } - |} - | - |def renderCLI(command, arguments) { - | def argumentsList = arguments.collect{renderArg(it)}.findAll{it != ""} - | - | def command_line = command + argumentsList - | - | return command_line.join(" ") - |} - | - |def effectiveContainer(processParams) { - | def _organization = params.containsKey("containerOrganization") ? params.containerOrganization : processParams.containerOrganization - | def _registry = params.containsKey("containerRegistry") ? params.containerRegistry : processParams.containerRegistry - | def _name = processParams.container - | def _tag = params.containsKey("containerTag") ? params.containerTag : processParams.containerTag - | - | return (_registry == "" ? "" : _registry + "/") + (_organization == "" ? "" : _organization + "/") + _name + ":" + _tag - |} - | - |// Convert the nextflow.config arguments list to a List instead of a LinkedHashMap - |// The rest of this main.nf script uses the Map form - |def argumentsAsList(_params) { - | def overrideArgs = _params.arguments.collect{ key, value -> value } - | def newParams = _params + [ "arguments" : overrideArgs ] - | return newParams - |} - | - |""".stripMargin - - val setup_main_outFromIn = - s""" - |// Use the params map, create a hashmap of the filenames for output - |// output filename is ..[.extension] - |def outFromIn(_params) { - | - | def id = _params.id - | - | _params - | .arguments - | .findAll{ it -> it.type == "file" && it.direction == "Output" } - | .collect{ it -> - | // If an 'example' attribute is present, strip the extension from the filename, - | // If a 'dflt' attribute is present, strip the extension from the filename, - | // Otherwise just use the option name as an extension. - | def extOrName = - | (it.example != null) - | ? it.example.split(/\\./).last() - | : (it.dflt != null) - | ? it.dflt.split(/\\./).last() - | : it.name - | // The output filename is . . - | // Unless the output argument is explicitly specified on the CLI - | def newValue = - | (it.value == "viash_no_value") - | ? "$fname." + it.name + "." + extOrName - | : it.value - | def newName = - | (id != "") - | ? id + "." + newValue - | : it.name + newValue - | it + [ value : newName ] - | } - | - |} - |""".stripMargin - - val setup_main_overrideIO = - """ - | - |def overrideIO(_params, inputs, outputs) { - | - | // `inputs` in fact can be one of: - | // - `String`, - | // - `List[String]`, - | // - `Map[String, String | List[String]]` - | // Please refer to the docs for more info - | def overrideArgs = _params.arguments.collect{ it -> - | if (it.type == "file") { - | if (it.direction == "Input") { - | (inputs in List || inputs in HashMap) - | ? (inputs in List) - | ? it + [ "value" : inputs.join(it.multiple_sep)] - | : (inputs[it.name] != null) - | ? (inputs[it.name] in List) - | ? it + [ "value" : inputs[it.name].join(it.multiple_sep)] - | : it + [ "value" : inputs[it.name]] - | : it - | : it + [ "value" : inputs ] - | } else { - | (outputs in List || outputs in HashMap) - | ? (outputs in List) - | ? it + [ "value" : outputs.join(it.multiple_sep)] - | : (outputs[it.name] != null) - | ? (outputs[it.name] in List) - | ? it + [ "value" : outputs[it.name].join(it.multiple_sep)] - | : it + [ "value" : outputs[it.name]] - | : it - | : it + [ "value" : outputs ] - | } - | } else { - | it - | } - | } - | - | def newParams = _params + [ "arguments" : overrideArgs ] - | - | return newParams - | - |} - |""".stripMargin - - def formatDirective(id: String, value: Option[String], delim: String): String = { - value.map(v => s"\n $id $delim$v$delim").getOrElse("") - } - - /** - * Some (implicit) conventions: - * - `params.output/` is where the output data is published - * - per_id is for creating directories per (sample) ID, default is true - * - path is for modifying the layout of the output directory, default is no changes - */ - val setup_main_process = { - - val per_idParsed = per_id.getOrElse(true) - val pathParsed = path.map(_.split("/").mkString("/") + "/").getOrElse("") - - // If id is the empty string, the subdirectory is not created - val publishDirString = - if (per_idParsed) { - s"$${params.publishDir}/$pathParsed$${id}/" - } else { - s"$${params.publishDir}/$pathParsed" - } - - val publishDirStr = publish match { - case Some(true) => s""" publishDir "$publishDirString", mode: 'copy', overwrite: true, enabled: !params.test""" - case _ => "" - } - - val directives = - labels.map(l => formatDirective("label", Some(l), "'")).mkString + - formatDirective("label", label, "'") + - formatDirective("cpus", directive_cpus.map(_.toString), "") + - formatDirective("maxForks", directive_max_forks.map(_.toString), "") + - formatDirective("time", directive_time, "'") + - formatDirective("memory", directive_memory, "'") + - formatDirective("cache", directive_cache, "'") - - val stageInModeStr = stageInMode match { - case Some("copy") => "copy" - case _ => "symlink" - } - - s""" - |process ${fname}_process {$directives - | tag "$${id}" - | echo { (params.debug == true) ? true : false } - | stageInMode "$stageInModeStr" - | container "$${container}" - |$publishDirStr - | input: - | tuple val(id), path(input), val(output), val(container), val(cli), val(_params) - | output: - | tuple val("$${id}"), path(output), val(_params) - | stub: - | \"\"\" - | # Adding NXF's `$$moduleDir` to the path in order to resolve our own wrappers - | export PATH="$${moduleDir}:\\$$PATH" - | STUB=1 $$cli - | \"\"\" - | script: - | def viash_temp = System.getenv("VIASH_TEMP") ?: "/tmp/" - | if (params.test) - | \"\"\" - | # Some useful stuff - | export NUMBA_CACHE_DIR=/tmp/numba-cache - | # Running the pre-hook when necessary - | # Pass viash temp dir - | export VIASH_TEMP="$${viash_temp}" - | # Adding NXF's `$$moduleDir` to the path in order to resolve our own wrappers - | export PATH="./:$${moduleDir}:\\$$PATH" - | ./$${params.$fname.tests.testScript} | tee $$output - | \"\"\" - | else - | \"\"\" - | # Some useful stuff - | export NUMBA_CACHE_DIR=/tmp/numba-cache - | # Running the pre-hook when necessary - | # Pass viash temp dir - | export VIASH_TEMP="$${viash_temp}" - | # Adding NXF's `$$moduleDir` to the path in order to resolve our own wrappers - | export PATH="$${moduleDir}:\\$$PATH" - | $$cli - | \"\"\" - |} - |""".stripMargin - } - - val emitter = - if (separate_multiple_outputs) { - s""" result_ - | | filter { it[1].keySet().size() > 1 } - | | view{">> Be careful, multiple outputs from this component!"} - | - | emit: - | result_.flatMap{ it -> - | (it[1].keySet().size() > 1) - | ? it[1].collect{ k, el -> [ it[0], [ (k): el ], it[2] ] } - | : it[1].collect{ k, el -> [ it[0], el, it[2] ] } - | }""".stripMargin - } else { - s""" emit: - | result_.flatMap{ it -> - | (it[1].keySet().size() > 1) - | ? [ it ] - | : it[1].collect{ k, el -> [ it[0], el, it[2] ] } - | }""".stripMargin - } - - val setup_main_workflow = - s""" - |workflow $fname { - | - | take: - | id_input_params_ - | - | main: - | - | def key = "$fname" - | - | def id_input_output_function_cli_params_ = - | id_input_params_.map{ id, input, _params -> - | - | // Start from the (global) params and overwrite with the (local) _params - | def defaultParams = params[key] ? params[key] : [:] - | def overrideParams = _params[key] ? _params[key] : [:] - | def updtParams = defaultParams + overrideParams - | // Convert to List[Map] for the arguments - | def newParams = argumentsAsList(updtParams) + [ "id" : id ] - | - | // Generate output filenames, out comes a Map - | def output = outFromIn(newParams) - | - | // The process expects Path or List[Path], Maps need to be converted - | def inputsForProcess = - | (input in HashMap) - | ? input.collect{ k, v -> v }.flatten() - | : input - | def outputsForProcess = output.collect{ it.value } - | - | // For our machinery, we convert Path -> String in the input - | def inputs = - | (input in List || input in HashMap) - | ? (input in List) - | ? input.collect{ it.name } - | : input.collectEntries{ k, v -> [ k, (v in List) ? v.collect{it.name} : v.name ] } - | : input.name - | outputs = output.collectEntries{ [(it.name): it.value] } - | - | def finalParams = overrideIO(newParams, inputs, outputs) - | - | checkParams(finalParams) - | - | new Tuple6( - | id, - | inputsForProcess, - | outputsForProcess, - | effectiveContainer(finalParams), - | renderCLI([finalParams.command], finalParams.arguments), - | finalParams - | ) - | } - | - | result_ = ${fname}_process(id_input_output_function_cli_params_) - | | join(id_input_params_) - | | map{ id, output, _params, input, original_params -> - | def parsedOutput = _params.arguments - | .findAll{ it.type == "file" && it.direction == "Output" } - | .withIndex() - | .collectEntries{ it, i -> - | // with one entry, output is of type Path and array selections - | // would select just one element from the path - | [(it.name): (output in List) ? output[i] : output ] - | } - | new Tuple3(id, parsedOutput, original_params) - | } - | - |${emitter.replaceAll("\n", "\n|")} - |} - |""".stripMargin - - val resultParseBlocks: List[String] = - if (separate_multiple_outputs && outputs >= 2) { - allPars - .filter(_.isInstanceOf[FileArgument]) - .filter(_.direction == Output) - .map(par => - s""" - | result \\ - | | filter${par.plainName.capitalize} \\ - | | view{ "Output for ${par.plainName}: " + it[1] } - |""".stripMargin - ) - } else { - List(" result.view{ it[1] }") - } - - val setup_main_entrypoint = - s""" - |workflow { - | def id = params.id - | def fname = "$fname" - | - | def _params = params - | - | // could be refactored to be FP - | for (entry in params[fname].arguments) { - | def name = entry.value.name - | if (params[name] != null) { - | params[fname].arguments[name].value = params[name] - | } - | } - | - | def inputFiles = params.$fname - | .arguments - | .findAll{ key, par -> par.type == "file" && par.direction == "Input" } - | .collectEntries{ key, par -> [(par.name): file(params[fname].arguments[par.name].value) ] } - | - | def ch_ = Channel.from("").map{ s -> new Tuple3(id, inputFiles, params)} - | - | result = $fname(ch_) - |${resultParseBlocks.mkString("\n").replaceAll("\n", "\n|")} - |} - |""".stripMargin - - val setup_test_entrypoint = - s""" - |// This workflow is not production-ready yet, we leave it in for future dev - |// TODO - |workflow test { - | - | take: - | rootDir - | - | main: - | params.test = true - | params.$fname.output = "$fname.log" - | - | Channel.from(rootDir) \\ - | | filter { params.$fname.tests.isDefined } \\ - | | map{ p -> new Tuple3( - | "tests", - | params.$fname.tests.testResources.collect{ file( p + it ) }, - | params - | )} \\ - | | $fname - | - | emit: - | $fname.out - |}""".stripMargin - - val setup_main = PlainFile( - dest = Some("main.nf"), - text = Some(setup_main_header + - setup_main_check + - setup_main_utils + - setup_main_outFromIn + - setup_main_outputFilters + - setup_main_overrideIO + - setup_main_process + - setup_main_workflow + - setup_main_entrypoint + - setup_test_entrypoint) - ) - - val additionalResources = mainResource match { - case None => Nil - case Some(_: Executable) => Nil - case Some(_: Script) => - nativePlatform.modifyFunctionality(config, false).resources - } - - functionality.copy( - resources = - additionalResources ::: List(setup_nextflowconfig, setup_main) - ) - } -} - -object NextFlowUtils { - import scala.reflect.runtime.universe._ - - def escapeNextflowString(s: String): String = { - Escaper( - s, - slash = true, - dollar = true, - backtick = false, - newline = true, - quote = true - ) - } - - def quote(str: String): String = s"\"$str\"" - - def quoteLong(str: String): String = str.replace("-", "_") - - trait ValueType - - case class PlainValue[A: TypeTag](v: A) extends ValueType { - def toConfig:String = v match { - case s: String if typeOf[String] =:= typeOf[A] => - quote(s) - case b: Boolean if typeOf[Boolean] =:= typeOf[A] => - b.toString - case c: Char if typeOf[Char] =:= typeOf[A] => - quote(c.toString) - case i: Int if typeOf[Int] =:= typeOf[A] => - i.toString - case l: List[_] => - l.map(el => quote(el.toString)).mkString("[ ", ", ", " ]") - case _ => - "Parsing ERROR - Not implemented yet " + v - } - } - - case class ConfigTuple(tuple: (String, ValueType)) { - def toConfig(indent: String = " "): String = { - val (k,v) = tuple - v match { - case pv: PlainValue[_] => - s"""$indent$k = ${pv.toConfig}""" - case NestedValue(nv) => - nv.map(_.toConfig(indent + " ")).mkString(s"$indent$k {\n", "\n", s"\n$indent}") - } - } - } - - case class NestedValue(v: List[ConfigTuple]) extends ValueType - - implicit def tupleToConfigTuple[A:TypeTag](tuple: (String, A)): ConfigTuple = { - val (k,v) = tuple - v match { - case NestedValue(nv) => ConfigTuple((k, NestedValue(nv))) - case _ => ConfigTuple((k, PlainValue(v))) - } - } - - def listMapToConfig(m: List[ConfigTuple]): String = { - m.map(_.toConfig()).mkString("\n") - } - - def namespacedValueTuple(key: String, value: String)(implicit fun: Functionality): ConfigTuple = - (s"${fun.name}__$key", value) - - implicit def argumentToConfigTuple[T:TypeTag](argument: Argument[T])(implicit fun: Functionality): ConfigTuple = { - val pointer = "${params." + fun.name + "__" + argument.plainName + "}" - - // TODO: Should this not be converted from the json? - val default = if (argument.default.isEmpty) None else Some(argument.default.mkString(argument.multiple_sep.toString)) - val example = if (argument.example.isEmpty) None else Some(argument.example.mkString(argument.multiple_sep.toString)) - quoteLong(argument.plainName) -> NestedValue( - tupleToConfigTuple("name" -> argument.plainName) :: - tupleToConfigTuple("flags" -> argument.flags) :: - tupleToConfigTuple("required" -> argument.required) :: - tupleToConfigTuple("type" -> argument.`type`) :: - tupleToConfigTuple("direction" -> argument.direction.toString) :: - tupleToConfigTuple("multiple" -> argument.multiple) :: - tupleToConfigTuple("multiple_sep" -> argument.multiple_sep) :: - tupleToConfigTuple("value" -> pointer) :: - default.map{ x => - List(tupleToConfigTuple("dflt" -> escapeNextflowString(x.toString))) - }.getOrElse(Nil) ::: - example.map{x => - List(tupleToConfigTuple("example" -> escapeNextflowString(x.toString))) - }.getOrElse(Nil) ::: - argument.description.map{x => - List(tupleToConfigTuple("description" -> escapeNextflowString(x.toString))) - }.getOrElse(Nil) - ) - } -} - -// vim: tabstop=2:softtabstop=2:shiftwidth=2:expandtab diff --git a/src/main/scala/io/viash/platforms/NextflowPlatform.scala b/src/main/scala/io/viash/platforms/NextflowPlatform.scala index 16a283aed..67fefa386 100644 --- a/src/main/scala/io/viash/platforms/NextflowPlatform.scala +++ b/src/main/scala/io/viash/platforms/NextflowPlatform.scala @@ -17,9 +17,332 @@ package io.viash.platforms -import io.viash.schemas.internalFunctionality +import io.viash.config.Config +import io.viash.functionality._ +import io.viash.functionality.resources._ +import io.viash.functionality.arguments._ +import io.viash.helpers.{Docker, Bash, DockerImageInfo, Helper} +import io.viash.helpers.circe._ +import io.viash.platforms.nextflow._ +import io.circe.syntax._ +import io.circe.{Printer => JsonPrinter, Json, JsonObject} +import shapeless.syntax.singleton +import io.viash.schemas._ +import io.viash.helpers.Escaper -trait NextflowPlatform extends Platform { - @internalFunctionality - val variant: String +/** + * A Platform class for generating Nextflow (DSL2) modules. + */ +@description("""Platform for generating Nextflow VDSL3 modules.""".stripMargin) +// todo: add link to guide +@example( + """platforms: + | - type: nextflow + | directives: + | label: [lowcpu, midmem] + |""".stripMargin, + "yaml") +@subclass("nextflow") +case class NextflowPlatform( + @description("Every platform can be given a specific id that can later be referred to explicitly when running or building the Viash component.") + @example("id: foo", "yaml") + @default("nextflow") + id: String = "nextflow", + + `type`: String = "nextflow", + + // nxf params + @description( + """@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts. + |""".stripMargin) + @example( + """directives: + | container: rocker/r-ver:4.1 + | label: highcpu + | cpus: 4 + | memory: 16 GB""".stripMargin, + "yaml") + @default("Empty") + directives: NextflowDirectives = NextflowDirectives(), + + @description( + """@[Automated processing flags](nextflow_auto) which can be toggled on or off: + | + || Flag | Description | Default | + ||---|---------|----| + || `simplifyInput` | If `true`, an input tuple only containing only a single File (e.g. `["foo", file("in.h5ad")]`) is automatically transformed to a map (i.e. `["foo", [ input: file("in.h5ad") ] ]`). | `true` | + || `simplifyOutput` | If `true`, an output tuple containing a map with a File (e.g. `["foo", [ output: file("out.h5ad") ] ]`) is automatically transformed to a map (i.e. `["foo", file("out.h5ad")]`). | `true` | + || `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` | + || `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. Will throw an error if `params.publishDir` is not defined. | `false` | + | + |""".stripMargin) + @example( + """auto: + | publish: true""".stripMargin, + "yaml") + @default( + """simplifyInput: true + |simplifyOutput: true + |transcript: false + |publish: false + |""".stripMargin) + auto: NextflowAuto = NextflowAuto(), + + @description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.") + @since("Viash 0.7.4") + @default("A series of default labels to specify memory and cpu constraints") + config: NextflowConfig = NextflowConfig(), + + @description("Whether or not to print debug messages.") + @default("False") + debug: Boolean = false, + + // TODO: solve differently + @description("Specifies the Docker platform id to be used to run Nextflow.") + @default("docker") + container: String = "docker" +) extends Platform { + def escapeSingleQuotedString(txt: String): String = { + Escaper(txt, slash = true, singleQuote = true, newline = true) + } + + def modifyFunctionality(config: Config, testing: Boolean): Functionality = { + val condir = containerDirective(config) + + // create main.nf file + val mainFile = PlainFile( + dest = Some("main.nf"), + text = Some(renderMainNf(config, condir)) + ) + val nextflowConfigFile = PlainFile( + dest = Some("nextflow.config"), + text = Some(renderNextflowConfig(config.functionality, condir)) + ) + + // remove main + val otherResources = config.functionality.additionalResources + + config.functionality.copy( + resources = mainFile :: nextflowConfigFile :: otherResources + ) + } + + def containerDirective(config: Config): Option[DockerImageInfo] = { + val plat = config.platforms.find(p => p.id == container) + plat match { + case Some(p: DockerPlatform) => + Some(Docker.getImageInfo( + functionality = Some(config.functionality), + registry = p.target_registry, + organization = p.target_organization, + name = p.target_image, + tag = p.target_tag.map(_.toString), + namespaceSeparator = p.namespace_separator + )) + case Some(_) => + throw new RuntimeException(s"NextflowPlatform 'container' variable: Platform $container is not a Docker Platform") + case None => None + } + } + + def renderNextflowConfig(functionality: Functionality, containerDirective: Option[DockerImageInfo]): String = { + val versStr = functionality.version.map(ver => s"\n version = '$ver'").getOrElse("") + + val descStr = functionality.description.map{des => + val escDes = escapeSingleQuotedString(des) + s"\n description = '$escDes'" + }.getOrElse("") + + val authStr = + if (functionality.authors.isEmpty) { + "" + } else { + val escAut = escapeSingleQuotedString(functionality.authors.map(_.name).mkString(", ")) + s"\n author = '$escAut'" + } + + // TODO: define profiles + val profileStr = + if (containerDirective.isDefined || functionality.mainScript.map(_.`type`) == Some(NextflowScript.`type`)) { + "\n\n" + NextflowHelper.profilesHelper + } else { + "" + } + + val processLabels = config.labels.map{ case (k, v) => s"withLabel: $k { $v }"} + val inlineScript = config.script.toList + + s"""manifest { + | name = '${functionality.name}' + | mainScript = 'main.nf' + | nextflowVersion = '!>=20.12.1-edge'$versStr$descStr$authStr + |}$profileStr + | + |process{ + | ${processLabels.mkString("\n ")} + |} + | + |${inlineScript.mkString("\n")} + |""".stripMargin + } + + // interpreted from BashWrapper + def renderMainNf(config: Config, containerDirective: Option[DockerImageInfo]): String = { + val functionality = config.functionality + + /************************* HEADER *************************/ + val header = Helper.generateScriptHeader(functionality) + .map(h => Escaper(h, newline = true)) + .mkString("// ", "\n// ", "") + + /************************* SCRIPT *************************/ + val executionCode = functionality.mainScript match { + // if mainResource is empty (shouldn't be the case) + case None => "" + + // if mainResource is simply an executable + case Some(e: Executable) => //" " + e.path.get + " $VIASH_EXECUTABLE_ARGS" + throw new NotImplementedError("Running executables through a NextflowPlatform is not yet implemented. Create a support ticket to request this functionality if necessary.") + + // if mainResource is a script + case Some(res) => + // todo: also include the bashwrapper checks + val argsAndMeta = functionality.getArgumentLikesGroupedByDest( + includeMeta = true, + filterInputs = true + ) + val code = res.readWithInjection(argsAndMeta).get + val escapedCode = Bash.escapeString(code, allowUnescape = true) + .replace("\\", "\\\\") + .replace("'''", "\\'\\'\\'") + + // IMPORTANT! difference between code below and BashWrapper: + // script is stored as `.viash_script.sh`. + val scriptPath = "$tempscript" + + s"""set -e + |tempscript=".viash_script.sh" + |cat > "$scriptPath" << VIASHMAIN + |$escapedCode + |VIASHMAIN + |${res.command(scriptPath)} + |""".stripMargin + } + + /************************* JSONS *************************/ + // override container + val directivesToJson = directives.copy( + // if a docker platform is defined but the directives.container isn't, use the image of the dockerplatform as default + container = directives.container orElse containerDirective.map(cd => Left(cd.toMap)), + // is memory requirements are defined but directives.memory isn't, use that instead + memory = directives.memory orElse functionality.requirements.memoryAsBytes.map(_.toString + " B"), + // is cpu requirements are defined but directives.cpus isn't, use that instead + cpus = directives.cpus orElse functionality.requirements.cpus.map(np => Left(np)) + ) + val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) + val dirJson = directivesToJson.asJson.dropEmptyRecursively + val dirJson2 = if (dirJson.isNull) Json.obj() else dirJson + val funJson = config.asJson.dropEmptyRecursively + val funJsonStr = jsonPrinter.print(funJson) + .replace("\\\\", "\\\\\\\\") + .replace("\\\"", "\\\\\"") + .replace("'''", "\\'\\'\\'") + .grouped(65000) // JVM has a maximum string limit of 65535 + .toList // see https://stackoverflow.com/a/6856773 + .mkString("'''", "''' + '''", "'''") + val autoJson = auto.asJson.dropEmptyRecursively + + /************************* MAIN.NF *************************/ + val tripQuo = """"""""" + + + s"""$header + | + |nextflow.enable.dsl=2 + | + |// Required imports + |import groovy.json.JsonSlurper + | + |// initialise slurper + |def jsonSlurper = new JsonSlurper() + | + |// DEFINE CUSTOM CODE + | + |// functionality metadata + |thisConfig = processConfig(jsonSlurper.parseText($funJsonStr)) + | + |thisScript = '''$executionCode''' + | + |thisDefaultProcessArgs = [ + | // key to be used to trace the process and determine output names + | key: thisConfig.functionality.name, + | // fixed arguments to be passed to script + | args: [:], + | // default directives + | directives: jsonSlurper.parseText('''${jsonPrinter.print(dirJson2)}'''), + | // auto settings + | auto: jsonSlurper.parseText('''${jsonPrinter.print(autoJson)}'''), + | + | // Apply a map over the incoming tuple + | // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + | map: null, + | + | // Apply a map over the ID element of a tuple (i.e. the first element) + | // Example: `{ id -> id + "_foo" }` + | mapId: null, + | + | // Apply a map over the data element of a tuple (i.e. the second element) + | // Example: `{ data -> [ input: data.output ] }` + | mapData: null, + | + | // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + | // Example: `{ pt -> pt.drop(1) }` + | mapPassthrough: null, + | + | // Filter the channel + | // Example: `{ tup -> tup[0] == "foo" }` + | filter: null, + | + | // Rename keys in the data field of the tuple (i.e. the second element) + | // Will likely be deprecated in favour of `fromState`. + | // Example: `[ "new_key": "old_key" ]` + | renameKeys: null, + | + | // Fetch data from the state and pass it to the module without altering the current state. + | // + | // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + | // + | // - If it is `null`, the state will be passed to the module as is. + | // - If it is a `List[String]`, the data will be the values of the state at the given keys. + | // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + | // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + | // + | // Example: `{ id, state -> [input: state.fastq_file] }` + | // Default: `null` + | fromState: null, + | + | // Determine how the state should be updated after the module has been run. + | // + | // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + | // + | // - If it is `null`, the state will be replaced with the output of the module. + | // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + | // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + | // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + | // + | // Example: `{ id, output, state -> state + [counts: state.output] }` + | // Default: `{ id, output, state -> output }` + | toState: null, + | + | // Whether or not to print debug messages + | // Default: `$debug` + | debug: $debug + |] + | + |// END CUSTOM CODE""".stripMargin + + "\n\n" + NextflowHelper.workflowHelper + + "\n\n" + NextflowHelper.vdsl3Helper + } } + +// vim: tabstop=2:softtabstop=2:shiftwidth=2:expandtab diff --git a/src/main/scala/io/viash/platforms/NextflowVdsl3Platform.scala b/src/main/scala/io/viash/platforms/NextflowVdsl3Platform.scala deleted file mode 100644 index a52792ce0..000000000 --- a/src/main/scala/io/viash/platforms/NextflowVdsl3Platform.scala +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Copyright (C) 2020 Data Intuitive - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package io.viash.platforms - -import io.viash.config.Config -import io.viash.functionality._ -import io.viash.functionality.resources._ -import io.viash.functionality.arguments._ -import io.viash.helpers.{Docker, Bash, DockerImageInfo, Helper} -import io.viash.helpers.circe._ -import io.viash.platforms.nextflow._ -import io.circe.syntax._ -import io.circe.{Printer => JsonPrinter, Json, JsonObject} -import shapeless.syntax.singleton -import io.viash.schemas._ -import io.viash.helpers.Escaper - -/** - * Next-gen Platform class for generating NextFlow (DSL2) modules. - */ -@description("Next-gen platform for generating NextFlow VDSL3 modules.") -@example( - """platforms: - | - type: nextflow - | directives: - | label: [lowcpu, midmem] - |""".stripMargin, - "yaml") -case class NextflowVdsl3Platform( - @description("Every platform can be given a specific id that can later be referred to explicitly when running or building the Viash component.") - @example("id: foo", "yaml") - id: String = "nextflow", - - `type`: String = "nextflow", - - @internalFunctionality - variant: String = "vdsl3", - - // nxf params - @description( - """@[Directives](nextflow_directives) are optional settings that affect the execution of the process. These mostly match up with the Nextflow counterparts. - |""".stripMargin) - @example( - """directives: - | container: rocker/r-ver:4.1 - | label: highcpu - | cpus: 4 - | memory: 16 GB""".stripMargin, - "yaml") - directives: NextflowDirectives = NextflowDirectives(), - - @description( - """@[Automated processing flags](nextflow_auto) which can be toggled on or off: - | - || Flag | Description | Default | - ||---|---------|----| - || `simplifyInput` | If `true`, an input tuple only containing only a single File (e.g. `["foo", file("in.h5ad")]`) is automatically transformed to a map (i.e. `["foo", [ input: file("in.h5ad") ] ]`). | `true` | - || `simplifyOutput` | If `true`, an output tuple containing a map with a File (e.g. `["foo", [ output: file("out.h5ad") ] ]`) is automatically transformed to a map (i.e. `["foo", file("out.h5ad")]`). | `true` | - || `transcript` | If `true`, the module's transcripts from `work/` are automatically published to `params.transcriptDir`. If not defined, `params.publishDir + "/_transcripts"` will be used. Will throw an error if neither are defined. | `false` | - || `publish` | If `true`, the module's outputs are automatically published to `params.publishDir`. Will throw an error if `params.publishDir` is not defined. | `false` | - | - |""".stripMargin) - @example( - """auto: - | publish: true""".stripMargin, - "yaml") - auto: NextflowAuto = NextflowAuto(), - - @description("Allows tweaking how the @[Nextflow Config](nextflow_config) file is generated.") - @since("Viash 0.7.4") - config: NextflowConfig = NextflowConfig(), - - @description("Whether or not to print debug messages.") - debug: Boolean = false, - - // TODO: solve differently - @description("Specifies the Docker platform id to be used to run Nextflow.") - container: String = "docker" -) extends NextflowPlatform { - def escapeSingleQuotedString(txt: String): String = { - Escaper(txt, slash = true, singleQuote = true, newline = true) - } - - def modifyFunctionality(config: Config, testing: Boolean): Functionality = { - val condir = containerDirective(config) - - // create main.nf file - val mainFile = PlainFile( - dest = Some("main.nf"), - text = Some(renderMainNf(config, condir)) - ) - val nextflowConfigFile = PlainFile( - dest = Some("nextflow.config"), - text = Some(renderNextflowConfig(config.functionality, condir)) - ) - - // remove main - val otherResources = config.functionality.additionalResources - - config.functionality.copy( - resources = mainFile :: nextflowConfigFile :: otherResources - ) - } - - def containerDirective(config: Config): Option[DockerImageInfo] = { - val plat = config.platforms.find(p => p.id == container) - plat match { - case Some(p: DockerPlatform) => - Some(Docker.getImageInfo( - functionality = Some(config.functionality), - registry = p.target_registry, - organization = p.target_organization, - name = p.target_image, - tag = p.target_tag.map(_.toString), - namespaceSeparator = p.namespace_separator - )) - case Some(_) => - throw new RuntimeException(s"NextflowPlatform 'container' variable: Platform $container is not a Docker Platform") - case None => None - } - } - - def renderNextflowConfig(functionality: Functionality, containerDirective: Option[DockerImageInfo]): String = { - val versStr = functionality.version.map(ver => s"\n version = '$ver'").getOrElse("") - - val descStr = functionality.description.map{des => - val escDes = escapeSingleQuotedString(des) - s"\n description = '$escDes'" - }.getOrElse("") - - val authStr = - if (functionality.authors.isEmpty) { - "" - } else { - val escAut = escapeSingleQuotedString(functionality.authors.map(_.name).mkString(", ")) - s"\n author = '$escAut'" - } - - // TODO: define profiles - val profileStr = - if (containerDirective.isDefined || functionality.mainScript.map(_.`type`) == Some(NextflowScript.`type`)) { - "\n\n" + NextflowHelper.profilesHelper - } else { - "" - } - - val processLabels = config.labels.map{ case (k, v) => s"withLabel: $k { $v }"} - val inlineScript = config.script.toList - - s"""manifest { - | name = '${functionality.name}' - | mainScript = 'main.nf' - | nextflowVersion = '!>=20.12.1-edge'$versStr$descStr$authStr - |}$profileStr - | - |process{ - | ${processLabels.mkString("\n ")} - |} - | - |${inlineScript.mkString("\n")} - |""".stripMargin - } - - // interpreted from BashWrapper - def renderMainNf(config: Config, containerDirective: Option[DockerImageInfo]): String = { - val functionality = config.functionality - - /************************* HEADER *************************/ - val header = Helper.generateScriptHeader(functionality) - .map(h => Escaper(h, newline = true)) - .mkString("// ", "\n// ", "") - - /************************* SCRIPT *************************/ - val executionCode = functionality.mainScript match { - // if mainResource is empty (shouldn't be the case) - case None => "" - - // if mainResource is simply an executable - case Some(e: Executable) => //" " + e.path.get + " $VIASH_EXECUTABLE_ARGS" - throw new NotImplementedError("Running executables through a NextflowPlatform is not yet implemented. Create a support ticket to request this functionality if necessary.") - - // if mainResource is a script - case Some(res) => - // todo: also include the bashwrapper checks - val argsAndMeta = functionality.getArgumentLikesGroupedByDest( - includeMeta = true, - filterInputs = true - ) - val code = res.readWithInjection(argsAndMeta).get - val escapedCode = Bash.escapeString(code, allowUnescape = true) - .replace("\\", "\\\\") - .replace("'''", "\\'\\'\\'") - - // IMPORTANT! difference between code below and BashWrapper: - // script is stored as `.viash_script.sh`. - val scriptPath = "$tempscript" - - s"""set -e - |tempscript=".viash_script.sh" - |cat > "$scriptPath" << VIASHMAIN - |$escapedCode - |VIASHMAIN - |${res.command(scriptPath)} - |""".stripMargin - } - - /************************* JSONS *************************/ - // override container - val directivesToJson = directives.copy( - // if a docker platform is defined but the directives.container isn't, use the image of the dockerplatform as default - container = directives.container orElse containerDirective.map(cd => Left(cd.toMap)), - // is memory requirements are defined but directives.memory isn't, use that instead - memory = directives.memory orElse functionality.requirements.memoryAsBytes.map(_.toString + " B"), - // is cpu requirements are defined but directives.cpus isn't, use that instead - cpus = directives.cpus orElse functionality.requirements.cpus.map(np => Left(np)) - ) - val jsonPrinter = JsonPrinter.spaces2.copy(dropNullValues = true) - val dirJson = directivesToJson.asJson.dropEmptyRecursively - val dirJson2 = if (dirJson.isNull) Json.obj() else dirJson - val funJson = config.asJson.dropEmptyRecursively - val funJsonStr = jsonPrinter.print(funJson) - .replace("\\\\", "\\\\\\\\") - .replace("\\\"", "\\\\\"") - .replace("'''", "\\'\\'\\'") - .grouped(65000) // JVM has a maximum string limit of 65535 - .toList // see https://stackoverflow.com/a/6856773 - .mkString("'''", "''' + '''", "'''") - val autoJson = auto.asJson.dropEmptyRecursively - - /************************* MAIN.NF *************************/ - val tripQuo = """"""""" - - - s"""$header - | - |nextflow.enable.dsl=2 - | - |// Required imports - |import groovy.json.JsonSlurper - | - |// initialise slurper - |def jsonSlurper = new JsonSlurper() - | - |// DEFINE CUSTOM CODE - | - |// functionality metadata - |thisConfig = processConfig(jsonSlurper.parseText($funJsonStr)) - | - |thisScript = '''$executionCode''' - | - |thisDefaultProcessArgs = [ - | // key to be used to trace the process and determine output names - | key: thisConfig.functionality.name, - | // fixed arguments to be passed to script - | args: [:], - | // default directives - | directives: jsonSlurper.parseText('''${jsonPrinter.print(dirJson2)}'''), - | // auto settings - | auto: jsonSlurper.parseText('''${jsonPrinter.print(autoJson)}'''), - | // apply a map over the incoming tuple - | // example: { tup -> [ tup[0], [input: tup[1].output], tup[2] ] } - | map: null, - | // apply a map over the ID element of a tuple (i.e. the first element) - | // example: { id -> id + "_foo" } - | mapId: null, - | // apply a map over the data element of a tuple (i.e. the second element) - | // example: { data -> [ input: data.output ] } - | mapData: null, - | // apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) - | // example: { pt -> pt.drop(1) } - | mapPassthrough: null, - | // filter the channel - | // example: { tup -> tup[0] == "foo" } - | filter: null, - | // rename keys in the data field of the tuple (i.e. the second element) - | // example: [ "new_key": "old_key" ] - | renameKeys: null, - | // whether or not to print debug messages - | debug: $debug - |] - | - |// END CUSTOM CODE""".stripMargin + - "\n\n" + NextflowHelper.workflowHelper + - "\n\n" + NextflowHelper.vdsl3Helper - } -} - -// vim: tabstop=2:softtabstop=2:shiftwidth=2:expandtab diff --git a/src/main/scala/io/viash/platforms/Platform.scala b/src/main/scala/io/viash/platforms/Platform.scala index d7f7bd72c..24fe3586a 100644 --- a/src/main/scala/io/viash/platforms/Platform.scala +++ b/src/main/scala/io/viash/platforms/Platform.scala @@ -30,7 +30,7 @@ import io.viash.schemas._ | | * @[Native](platform_native) | * @[Docker](platform_docker) - | * @[Nextflow VDSL3](platform_nextflow) + | * @[Nextflow](platform_nextflow) |""".stripMargin) @example( """platforms: @@ -42,6 +42,9 @@ import io.viash.schemas._ | label: [lowcpu, midmem] |""".stripMargin, "yaml") +@subclass("NativePlatform") +@subclass("DockerPlatform") +@subclass("NextflowPlatform") trait Platform { @description("Specifies the type of the platform.") val `type`: String diff --git a/src/main/scala/io/viash/platforms/nextflow/NextflowAuto.scala b/src/main/scala/io/viash/platforms/nextflow/NextflowAuto.scala index f30d8b13b..29134fb4c 100644 --- a/src/main/scala/io/viash/platforms/nextflow/NextflowAuto.scala +++ b/src/main/scala/io/viash/platforms/nextflow/NextflowAuto.scala @@ -17,7 +17,7 @@ package io.viash.platforms.nextflow -import io.viash.schemas.description +import io.viash.schemas._ @description("Automated processing flags which can be toggled on or off.") case class NextflowAuto( @@ -26,6 +26,7 @@ case class NextflowAuto( | |Default: `true`. |""".stripMargin) + @default("True") simplifyInput: Boolean = true, @description( @@ -33,6 +34,7 @@ case class NextflowAuto( | |Default: `true`. |""".stripMargin) + @default("True") simplifyOutput: Boolean = true, @description( @@ -42,6 +44,7 @@ case class NextflowAuto( | |Default: `false`. |""".stripMargin) + @default("False") transcript: Boolean = false, @description( @@ -50,5 +53,6 @@ case class NextflowAuto( | |Default: `false`. |""".stripMargin) + @default("False") publish: Boolean = false, ) \ No newline at end of file diff --git a/src/main/scala/io/viash/platforms/nextflow/NextflowConfig.scala b/src/main/scala/io/viash/platforms/nextflow/NextflowConfig.scala index 15f4677ec..e4fe00307 100644 --- a/src/main/scala/io/viash/platforms/nextflow/NextflowConfig.scala +++ b/src/main/scala/io/viash/platforms/nextflow/NextflowConfig.scala @@ -65,6 +65,7 @@ case class NextflowConfig( "viash_project_file", "Replace the default labels with a different set of labels by using the Viash Project file" ) + @default("A series of default labels to specify memory and cpu constraints") labels: ListMap[String, String] = ListMap( NextflowConfig.binaryIterator .dropWhile(_ < 1 * NextflowConfig.GB) @@ -92,6 +93,7 @@ case class NextflowConfig( |""".stripMargin, "yaml") @example("""script: includeConfig("config.config")""", "yaml") + @default("Empty") script: OneOrMore[String] = Nil ) diff --git a/src/main/scala/io/viash/platforms/nextflow/NextflowDirectives.scala b/src/main/scala/io/viash/platforms/nextflow/NextflowDirectives.scala index caa66b75b..cadb940eb 100644 --- a/src/main/scala/io/viash/platforms/nextflow/NextflowDirectives.scala +++ b/src/main/scala/io/viash/platforms/nextflow/NextflowDirectives.scala @@ -40,6 +40,7 @@ case class NextflowDirectives( |See [`accelerator`](https://www.nextflow.io/docs/latest/process.html#accelerator). |""".stripMargin) @example("""[ limit: 4, type: "nvidia-tesla-k80" ]""", "yaml") + @default("Empty") accelerator: Map[String, String] = Map(), @description( @@ -85,6 +86,7 @@ case class NextflowDirectives( @example(""""bwa=0.7.15"""", "yaml") @example(""""bwa=0.7.15 fastqc=0.11.5"""", "yaml") @example("""["bwa=0.7.15", "fastqc=0.11.5"]""", "yaml") + @default("Empty") conda: OneOrMore[String] = Nil, @description( @@ -108,6 +110,7 @@ case class NextflowDirectives( |""".stripMargin) @example(""""--foo bar"""", "yaml") @example("""["--foo bar", "-f b"]""", "yaml") + @default("Empty") containerOptions: OneOrMore[String] = Nil, @description( @@ -195,6 +198,7 @@ case class NextflowDirectives( @example(""""big_mem"""", "yaml") @example(""""big_cpu"""", "yaml") @example("""["big_mem", "big_cpu"]""", "yaml") + @default("Empty") label: OneOrMore[String] = Nil, @description( @@ -257,6 +261,7 @@ case class NextflowDirectives( @example(""""ncbi-blast/2.2.27"""", "yaml") @example(""""ncbi-blast/2.2.27:t_coffee/10.0"""", "yaml") @example("""["ncbi-blast/2.2.27", "t_coffee/10.0"]""", "yaml") + @default("Empty") module: OneOrMore[String] = Nil, @description( @@ -276,6 +281,7 @@ case class NextflowDirectives( @example("""[ annotation: "key", value: "val" ]""", "yaml") @example("""[ env: "key", value: "val" ]""", "yaml") @example("""[ [label: "l", value: "v"], [env: "e", value: "v"]]""", "yaml") + @default("Empty") pod: OneOrMore[Map[String, String]] = Nil, @description( @@ -290,6 +296,7 @@ case class NextflowDirectives( @example("""[ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ]""", "yaml") @exampleWithDescription(""""/path/to/dir"""", "yaml", """This is transformed to `[[ path: "/path/to/dir" ]]`:""") @exampleWithDescription("""[ path: "/path/to/dir", mode: "cache" ]""", "yaml", """This is transformed to `[[ path: "/path/to/dir", mode: "cache" ]]`:""") + @default("Empty") publishDir: OneOrMore[Either[String, Map[String, String]]] = Nil, // TODO: need to implement publishdir class? @description( @@ -300,6 +307,7 @@ case class NextflowDirectives( @example(""""long"""", "yaml") @example(""""short,long"""", "yaml") @example("""["short", "long"]""", "yaml") + @default("Empty") queue: OneOrMore[String] = Nil, @description( diff --git a/src/main/scala/io/viash/platforms/nextflow/package.scala b/src/main/scala/io/viash/platforms/nextflow/package.scala index fdb8d57ce..fca12c85f 100644 --- a/src/main/scala/io/viash/platforms/nextflow/package.scala +++ b/src/main/scala/io/viash/platforms/nextflow/package.scala @@ -22,14 +22,14 @@ import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfigur package object nextflow { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ implicit val encodeNextflowDirectives: Encoder.AsObject[NextflowDirectives] = deriveConfiguredEncoder - implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeNextflowDirectives: Decoder[NextflowDirectives] = deriveConfiguredDecoderFullChecks implicit val encodeNextflowAuto: Encoder.AsObject[NextflowAuto] = deriveConfiguredEncoder - implicit val decodeNextflowAuto: Decoder[NextflowAuto] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeNextflowAuto: Decoder[NextflowAuto] = deriveConfiguredDecoderFullChecks implicit val encodeNextflowConfig: Encoder.AsObject[NextflowConfig] = deriveConfiguredEncoder - implicit val decodeNextflowConfig: Decoder[NextflowConfig] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeNextflowConfig: Decoder[NextflowConfig] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/platforms/package.scala b/src/main/scala/io/viash/platforms/package.scala index d4c272a38..f8da40db6 100644 --- a/src/main/scala/io/viash/platforms/package.scala +++ b/src/main/scala/io/viash/platforms/package.scala @@ -23,45 +23,28 @@ import cats.syntax.functor._ // for .widen package object platforms { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ implicit val encodeDockerPlatform: Encoder.AsObject[DockerPlatform] = deriveConfiguredEncoder - implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeDockerPlatform: Decoder[DockerPlatform] = deriveConfiguredDecoderFullChecks - implicit val encodeNextflowLegacyPlatform: Encoder.AsObject[NextflowLegacyPlatform] = deriveConfiguredEncoder - implicit val decodeNextflowLegacyPlatform: Decoder[NextflowLegacyPlatform] = deriveConfiguredDecoderWithDeprecationCheck - - implicit val encodeNextflowVdsl3Platform: Encoder.AsObject[NextflowVdsl3Platform] = deriveConfiguredEncoder - implicit val decodeNextflowVdsl3Platform: Decoder[NextflowVdsl3Platform] = deriveConfiguredDecoderWithDeprecationCheck + implicit val encodeNextflowPlatform: Encoder.AsObject[NextflowPlatform] = deriveConfiguredEncoder + implicit val decodeNextflowPlatform: Decoder[NextflowPlatform] = deriveConfiguredDecoderFullChecks implicit val encodeNativePlatform: Encoder.AsObject[NativePlatform] = deriveConfiguredEncoder - implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeNativePlatform: Decoder[NativePlatform] = deriveConfiguredDecoderFullChecks implicit def encodePlatform[A <: Platform]: Encoder[A] = Encoder.instance { platform => val typeJson = Json.obj("type" -> Json.fromString(platform.`type`)) val objJson = platform match { case s: DockerPlatform => encodeDockerPlatform(s) - // case s: NextflowLegacyPlatform => encodeNextflowLegacyPlatform(s) - case s: NextflowVdsl3Platform => encodeNextflowVdsl3Platform(s) + case s: NextflowPlatform => encodeNextflowPlatform(s) case s: NativePlatform => encodeNativePlatform(s) } objJson deepMerge typeJson } - implicit def decodeNextflowPlatform: Decoder[NextflowPlatform] = Decoder.instance { - cursor => - val decoder: Decoder[NextflowPlatform] = - cursor.downField("variant").as[String] match { - // case Right("legacy") => decodeNextflowLegacyPlatform.widen - case Right("vdsl3") => decodeNextflowVdsl3Platform.widen - case Right(typ) => throw new RuntimeException("Variant " + typ + " is not recognised.") - case Left(exception) => decodeNextflowVdsl3Platform.widen - } - - decoder(cursor) - } - implicit def decodePlatform: Decoder[Platform] = Decoder.instance { cursor => val decoder: Decoder[Platform] = @@ -69,7 +52,9 @@ package object platforms { case Right("docker") => decodeDockerPlatform.widen case Right("native") => decodeNativePlatform.widen case Right("nextflow") => decodeNextflowPlatform.widen - case Right(typ) => throw new RuntimeException("Type " + typ + " is not recognised.") + case Right(typ) => + //throw new RuntimeException("Type " + typ + " is not recognised.") + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[NativePlatform](typ, List("docker", "native", "nextflow")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/platforms/requirements/ApkRequirements.scala b/src/main/scala/io/viash/platforms/requirements/ApkRequirements.scala index 62b9c354b..37e1cab87 100644 --- a/src/main/scala/io/viash/platforms/requirements/ApkRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/ApkRequirements.scala @@ -27,9 +27,11 @@ import io.viash.schemas._ | packages: [ sl ] |""".stripMargin, "yaml") +@subclass("apk") case class ApkRequirements( @description("Specifies which packages to install.") @example("packages: [ sl ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, `type`: String = "apk" diff --git a/src/main/scala/io/viash/platforms/requirements/AptRequirements.scala b/src/main/scala/io/viash/platforms/requirements/AptRequirements.scala index cb899fcae..255cab2ba 100644 --- a/src/main/scala/io/viash/platforms/requirements/AptRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/AptRequirements.scala @@ -27,12 +27,15 @@ import io.viash.schemas._ | packages: [ sl ] |""".stripMargin, "yaml") +@subclass("apt") case class AptRequirements( @description("Specifies which packages to install.") @example("packages: [ sl ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, @description("If `false`, the Debian frontend is set to non-interactive (recommended). Default: false.") + @default("False") interactive: Boolean = false, `type`: String = "apt" ) extends Requirements { diff --git a/src/main/scala/io/viash/platforms/requirements/DockerRequirements.scala b/src/main/scala/io/viash/platforms/requirements/DockerRequirements.scala index e74916a32..1dbde9b94 100644 --- a/src/main/scala/io/viash/platforms/requirements/DockerRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/DockerRequirements.scala @@ -29,41 +29,42 @@ import io.viash.schemas._ # echo 'Run a custom command' # echo 'Foo' > /path/to/file.txt""".stripMargin('#'), "yaml") +@subclass("docker") case class DockerRequirements( @description("Specifies which `LABEL` entries to add to the Dockerfile while building it.") @example("label: [ component=\"foo\" ]", "yaml") + @default("Empty") label: OneOrMore[String] = Nil, @description("Specifies which `ADD` entries to add to the Dockerfile while building it.") @example("add: [ \"http://foo/bar .\" ]", "yaml") + @default("Empty") add: OneOrMore[String] = Nil, @description("Specifies which `COPY` entries to add to the Dockerfile while building it.") @example("copy: [ \"resource.txt /path/to/resource.txt\" ]", "yaml") + @default("Empty") copy: OneOrMore[String] = Nil, @description("Specifies which `RUN` entries to add to the Dockerfile while building it.") @example("""run: | # echo 'Run a custom command' # echo 'Foo' > /path/to/file.txt""".stripMargin('#'), "yaml") + @default("Empty") run: OneOrMore[String] = Nil, @description("Specifies which `ARG` entries to add to the Dockerfile while building it.") @example("build_args: [ \"R_VERSION=4.2\" ]", "yaml") + @default("Empty") build_args: OneOrMore[String] = Nil, @description("Specifies which `ENV` entries to add to the Dockerfile while building it. Unlike `ARG`, `ENV` entries are also accessible from inside the container.") @example("env: [ \"R_VERSION=4.2\" ]", "yaml") + @default("Empty") env: OneOrMore[String] = Nil, `type`: String = "docker" ) extends Requirements { -// START OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED - @description("Specifies which `COPY` entries to add to the Dockerfile while building it.") - @example("resources: [ \"resource.txt /path/to/resource.txt\" ]", "yaml") - @removed("`resources` in `setup: {type: docker, resources: ...}` was removed. Please use `copy` instead.", "0.6.3", "0.7.0") - private val resources: OneOrMore[String] = Nil - // END OF REMOVED PARAMETERS THAT ARE STILL DOCUMENTED def installCommands: List[String] = Nil @@ -96,13 +97,6 @@ case class DockerRequirements( Nil } - val resourcess = - if (resources.nonEmpty) { - resources.map(c => s"""COPY $c""") - } else { - Nil - } - val envs = if (env.nonEmpty) { env.map(c => s"""ENV $c""") @@ -117,7 +111,7 @@ case class DockerRequirements( Nil } - val li = args ::: labels ::: envs ::: copys ::: resourcess ::: adds ::: runCommands + val li = args ::: labels ::: envs ::: copys ::: adds ::: runCommands if (li.isEmpty) None else Some(li.mkString("\n")) } diff --git a/src/main/scala/io/viash/platforms/requirements/JavaScriptRequirements.scala b/src/main/scala/io/viash/platforms/requirements/JavaScriptRequirements.scala index 63aa2d077..78599ef3a 100644 --- a/src/main/scala/io/viash/platforms/requirements/JavaScriptRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/JavaScriptRequirements.scala @@ -30,25 +30,31 @@ import io.viash.schemas._ | url: "https://github.com/org/repo/archive/HEAD.zip" |""".stripMargin, "yaml") +@subclass("javascript") case class JavaScriptRequirements( @description("Specifies which packages to install from npm.") @example("packages: [ packagename ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, @description("Specifies which packages to install from npm.") @example("npm: [ packagename ]", "yaml") + @default("Empty") npm: OneOrMore[String] = Nil, @description("Specifies which packages to install using a Git URI.") @example("git: [ https://some.git.repository/org/repo ]", "yaml") + @default("Empty") git: OneOrMore[String] = Nil, @description("Specifies which packages to install from GitHub.") @example("github: [ owner/repository ]", "yaml") + @default("Empty") github: OneOrMore[String] = Nil, @description("Specifies which packages to install using a generic URI.") @example("url: [ https://github.com/org/repo/archive/HEAD.zip ]", "yaml") + @default("Empty") url: OneOrMore[String] = Nil, `type`: String = "javascript" ) extends Requirements { diff --git a/src/main/scala/io/viash/platforms/requirements/PythonRequirements.scala b/src/main/scala/io/viash/platforms/requirements/PythonRequirements.scala index d0c2bd5fe..b4b575e6c 100644 --- a/src/main/scala/io/viash/platforms/requirements/PythonRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/PythonRequirements.scala @@ -29,57 +29,71 @@ import io.viash.schemas._ | url: "https://github.com/some_org/some_pkg/zipball/master" |""".stripMargin, "yaml") +@subclass("python") case class PythonRequirements( @description("Sets the `--user` flag when set to true. Default: false.") + @default("False") user: Boolean = false, @description("Specifies which packages to install from pip.") @example("packages: [ numpy ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, @description("Specifies which packages to install from pip.") @example("pip: [ numpy ]", "yaml") + @default("Empty") pip: OneOrMore[String] = Nil, @description("Specifies which packages to install from PyPI using pip.") @example("pypi: [ numpy ]", "yaml") + @default("Empty") pypi: OneOrMore[String] = Nil, @description("Specifies which packages to install using a Git URI.") @example("git: [ https://some.git.repository/org/repo ]", "yaml") + @default("Empty") git: OneOrMore[String] = Nil, @description("Specifies which packages to install from GitHub.") @example("github: [ jkbr/httpie ]", "yaml") + @default("Empty") github: OneOrMore[String] = Nil, @description("Specifies which packages to install from GitLab.") @example("gitlab: [ foo/bar ]", "yaml") + @default("Empty") gitlab: OneOrMore[String] = Nil, @description("Specifies which packages to install using a Mercurial URI.") @example("mercurial: [ https://hg.myproject.org/MyProject/#egg=MyProject ]", "yaml") + @default("Empty") mercurial: OneOrMore[String] = Nil, @description("Specifies which packages to install using an SVN URI.") @example("svn: [ http://svn.repo/some_pkg/trunk/#egg=SomePackage ]", "yaml") + @default("Empty") svn: OneOrMore[String] = Nil, @description("Specifies which packages to install using a Bazaar URI.") @example("bazaar: [ http://bazaar.launchpad.net/some_pkg/some_pkg/release-0.1 ]", "yaml") + @default("Empty") bazaar: OneOrMore[String] = Nil, @description("Specifies which packages to install using a generic URI.") @example("url: [ https://github.com/some_org/some_pkg/zipball/master ]", "yaml") + @default("Empty") url: OneOrMore[String] = Nil, @description("Specifies a code block to run as part of the build.") @example("""script: | # print("Running custom code") # x = 1 + 1 == 2""".stripMargin('#'), "yaml") + @default("Empty") script: OneOrMore[String] = Nil, @description("Sets the `--upgrade` flag when set to true. Default: true.") + @default("True") upgrade: Boolean = true, `type`: String = "python" ) extends Requirements { diff --git a/src/main/scala/io/viash/platforms/requirements/RRequirements.scala b/src/main/scala/io/viash/platforms/requirements/RRequirements.scala index 203521253..eb383715e 100644 --- a/src/main/scala/io/viash/platforms/requirements/RRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/RRequirements.scala @@ -29,51 +29,63 @@ import io.viash.schemas._ | github: rcannood/SCORPIUS |""".stripMargin, "yaml") +@subclass("r") case class RRequirements( @description("Specifies which packages to install from CRAN.") @example("packages: [ anndata, ggplot2 ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, @description("Specifies which packages to install from CRAN.") @example("cran: [ anndata, ggplot2 ]", "yaml") + @default("Empty") cran: OneOrMore[String] = Nil, @description("Specifies which packages to install from BioConductor.") @example("bioc: [ AnnotationDbi ]", "yaml") + @default("Empty") bioc: OneOrMore[String] = Nil, @description("Specifies which packages to install using a Git URI.") @example("git: [ https://some.git.repository/org/repo ]", "yaml") + @default("Empty") git: OneOrMore[String] = Nil, @description("Specifies which packages to install from GitHub.") @example("github: [ rcannood/SCORPIUS ]", "yaml") + @default("Empty") github: OneOrMore[String] = Nil, @description("Specifies which packages to install from GitLab.") @example("gitlab: [ org/package ]", "yaml") + @default("Empty") gitlab: OneOrMore[String] = Nil, @description("Specifies which packages to install from Bitbucket.") @example("bitbucket: [ org/package ]", "yaml") + @default("Empty") bitbucket: OneOrMore[String] = Nil, @description("Specifies which packages to install using an SVN URI.") @example("svn: [ https://path.to.svn/group/repo ]", "yaml") + @default("Empty") svn: OneOrMore[String] = Nil, @description("Specifies which packages to install using a generic URI.") @example("url: [ https://github.com/hadley/stringr/archive/HEAD.zip ]", "yaml") + @default("Empty") url: OneOrMore[String] = Nil, @description("Specifies a code block to run as part of the build.") @example("""script: | # cat("Running custom code\n") # install.packages("anndata")""".stripMargin('#'), "yaml") + @default("Empty") script: OneOrMore[String] = Nil, @description("Forces packages specified in `bioc` to be reinstalled, even if they are already present in the container. Default: false.") @example("bioc_force_install: false", "yaml") + @default("False") bioc_force_install: Boolean = false, `type`: String = "r" diff --git a/src/main/scala/io/viash/platforms/requirements/Requirements.scala b/src/main/scala/io/viash/platforms/requirements/Requirements.scala index a8b3cd38e..2668ab1e0 100644 --- a/src/main/scala/io/viash/platforms/requirements/Requirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/Requirements.scala @@ -17,7 +17,7 @@ package io.viash.platforms.requirements -import io.viash.schemas.description +import io.viash.schemas._ @description( """Requirements for installing the following types of packages: @@ -31,6 +31,14 @@ import io.viash.schemas.description | - @[Ruby](ruby_req) | - @[yum](yum_req) |""".stripMargin) +@subclass("ApkRequirements") +@subclass("AptRequirements") +@subclass("DockerRequirements") +@subclass("JavaScriptRequirements") +@subclass("PythonRequirements") +@subclass("RRequirements") +@subclass("RubyRequirements") +@subclass("YumRequirements") trait Requirements { @description("Specifies the type of the requirement specification.") val `type`: String diff --git a/src/main/scala/io/viash/platforms/requirements/RubyRequirements.scala b/src/main/scala/io/viash/platforms/requirements/RubyRequirements.scala index 9b310c708..8e72229ed 100644 --- a/src/main/scala/io/viash/platforms/requirements/RubyRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/RubyRequirements.scala @@ -27,9 +27,11 @@ import io.viash.schemas._ | packages: [ rspec ] |""".stripMargin, "yaml") +@subclass("ruby") case class RubyRequirements( @description("Specifies which packages to install.") @example("packages: [ rspec ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, `type`: String = "ruby" diff --git a/src/main/scala/io/viash/platforms/requirements/YumRequirements.scala b/src/main/scala/io/viash/platforms/requirements/YumRequirements.scala index 8e057d672..2ec316da0 100644 --- a/src/main/scala/io/viash/platforms/requirements/YumRequirements.scala +++ b/src/main/scala/io/viash/platforms/requirements/YumRequirements.scala @@ -27,9 +27,11 @@ import io.viash.schemas._ | packages: [ sl ] |""".stripMargin, "yaml") +@subclass("yum") case class YumRequirements( @description("Specifies which packages to install.") @example("packages: [ sl ]", "yaml") + @default("Empty") packages: OneOrMore[String] = Nil, `type`: String = "yum" diff --git a/src/main/scala/io/viash/platforms/requirements/package.scala b/src/main/scala/io/viash/platforms/requirements/package.scala index 903c0a012..ca777d184 100644 --- a/src/main/scala/io/viash/platforms/requirements/package.scala +++ b/src/main/scala/io/viash/platforms/requirements/package.scala @@ -25,31 +25,31 @@ import cats.syntax.functor._ // for .widen package object requirements { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ implicit val encodeRRequirements: Encoder.AsObject[RRequirements] = deriveConfiguredEncoder - implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeRRequirements: Decoder[RRequirements] = deriveConfiguredDecoderFullChecks implicit val encodePythonRequirements: Encoder.AsObject[PythonRequirements] = deriveConfiguredEncoder - implicit val decodePythonRequirements: Decoder[PythonRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodePythonRequirements: Decoder[PythonRequirements] = deriveConfiguredDecoderFullChecks implicit val encodeRubyRequirements: Encoder.AsObject[RubyRequirements] = deriveConfiguredEncoder - implicit val decodeRubyRequirements: Decoder[RubyRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeRubyRequirements: Decoder[RubyRequirements] = deriveConfiguredDecoderFullChecks implicit val encodeJavaScriptRequirements: Encoder.AsObject[JavaScriptRequirements] = deriveConfiguredEncoder - implicit val decodeJavaScriptRequirements: Decoder[JavaScriptRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeJavaScriptRequirements: Decoder[JavaScriptRequirements] = deriveConfiguredDecoderFullChecks implicit val encodeAptRequirements: Encoder.AsObject[AptRequirements] = deriveConfiguredEncoder - implicit val decodeAptRequirements: Decoder[AptRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeAptRequirements: Decoder[AptRequirements] = deriveConfiguredDecoderFullChecks implicit val encodeYumRequirements: Encoder.AsObject[YumRequirements] = deriveConfiguredEncoder - implicit val decodeYumRequirements: Decoder[YumRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeYumRequirements: Decoder[YumRequirements] = deriveConfiguredDecoderFullChecks implicit val encodeApkRequirements: Encoder.AsObject[ApkRequirements] = deriveConfiguredEncoder - implicit val decodeApkRequirements: Decoder[ApkRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeApkRequirements: Decoder[ApkRequirements] = deriveConfiguredDecoderFullChecks implicit val encodeDockerRequirements: Encoder.AsObject[DockerRequirements] = deriveConfiguredEncoder - implicit val decodeDockerRequirements: Decoder[DockerRequirements] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeDockerRequirements: Decoder[DockerRequirements] = deriveConfiguredDecoderFullChecks implicit def encodeRequirements[A <: Requirements]: Encoder[A] = Encoder.instance { reqs => @@ -79,7 +79,8 @@ package object requirements { case Right("r") => decodeRRequirements.widen case Right("javascript") => decodeJavaScriptRequirements.widen case Right("ruby") => decodeRubyRequirements.widen - case Right(typ) => throw new RuntimeException("Type " + typ + " is not recognised.") + case Right(typ) => + DeriveConfiguredDecoderWithValidationCheck.invalidSubTypeDecoder[ApkRequirements](typ, List("apk", "apt", "yum", "docker", "python", "r", "javascript", "ruby")).widen case Left(exception) => throw exception } diff --git a/src/main/scala/io/viash/project/ViashProject.scala b/src/main/scala/io/viash/project/ViashProject.scala index b04533443..dd7cfd955 100644 --- a/src/main/scala/io/viash/project/ViashProject.scala +++ b/src/main/scala/io/viash/project/ViashProject.scala @@ -17,15 +17,12 @@ package io.viash.project -import java.nio.file.{Files, Path} - -import io.circe.yaml.parser +import java.nio.file.{Files, Path, Paths} import io.viash.schemas._ import io.viash.helpers.data_structures.OneOrMore import io.viash.helpers.IO import io.viash.helpers.circe._ -import java.nio.file.Paths import io.circe.Json import java.net.URI @@ -42,6 +39,7 @@ import java.net.URI §""".stripMargin('§'), "yaml" ) @since("Viash 0.6.4") +@nameOverride("Project") case class ViashProject( @description("Which version of Viash to use.") @example("viash_versions: 0.6.4", "yaml") @@ -61,6 +59,7 @@ case class ViashProject( // todo: link to config mods docs @description("Which config mods to apply.") @example("config_mods: \".functionality.name := 'foo'\"", "yaml") + @default("Empty") config_mods: OneOrMore[String] = Nil, @description("Directory in which the _viash.yaml resides.") @@ -90,14 +89,6 @@ object ViashProject { } } - private def parsingErrorHandler[C](uri: Option[URI]) = { - (e: Exception) => { - val uriStr = uri.map(u => s" '{u}'").getOrElse("") - Console.err.println(s"${Console.RED}Error parsing$uriStr.${Console.RESET}\nDetails:") - throw e - } - } - /** * Read the text from a Path and convert to a Json * @@ -110,7 +101,7 @@ object ViashProject { // read yaml as string val projStr = IO.read(uri) - val json0 = parser.parse(projStr).fold(parsingErrorHandler(Some(uri)), identity) + val json0 = Convert.textToJson(projStr, path.toString()) /* JSON 1: after inheritance */ // apply inheritance if need be @@ -132,8 +123,7 @@ object ViashProject { /* PROJECT 0: converted from json */ // convert Json into ViashProject - val proj0 = json.as[ViashProject].fold(parsingErrorHandler(Some(path.toUri())), identity) - + val proj0 = Convert.jsonToClass[ViashProject](json, path.toString()) /* PROJECT 1: make resources absolute */ // make paths absolute diff --git a/src/main/scala/io/viash/project/package.scala b/src/main/scala/io/viash/project/package.scala index e04943eef..cc3885bc8 100644 --- a/src/main/scala/io/viash/project/package.scala +++ b/src/main/scala/io/viash/project/package.scala @@ -22,8 +22,8 @@ import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfigur package object project { import io.viash.helpers.circe._ - import io.viash.helpers.circe.DeriveConfiguredDecoderWithDeprecationCheck._ + import io.viash.helpers.circe.DeriveConfiguredDecoderFullChecks._ implicit val encodeViashProject: Encoder.AsObject[ViashProject] = deriveConfiguredEncoder - implicit val decodeViashProject: Decoder[ViashProject] = deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeViashProject: Decoder[ViashProject] = deriveConfiguredDecoderFullChecks } diff --git a/src/main/scala/io/viash/schemas/AutoComplete.scala b/src/main/scala/io/viash/schemas/AutoComplete.scala new file mode 100644 index 000000000..73e90ebe9 --- /dev/null +++ b/src/main/scala/io/viash/schemas/AutoComplete.scala @@ -0,0 +1,259 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.schemas + +import io.viash.cli._ + +object AutoCompleteBash { + def commandArguments(cmd: RegisteredCommand): String = { + val (opts, trailOpts) = cmd.opts.partition(_.optType != "trailArgs") + val optNames = opts.map(_.name) ++ Seq("help") + val cmdName = cmd.name + + trailOpts match { + case Nil => + s"""$cmdName) + | COMPREPLY=($$(compgen -W ${optNames.mkString("'--", " --", "'")} -- "$$cur")) + | return + | ;; + |""".stripMargin + case _ => + s"""$cmdName) + | if [[ $$cur == -* ]]; then + | COMPREPLY=($$(compgen -W ${optNames.mkString("'--", " --", "'")} -- "$$cur")) + | return + | fi + | _filedir + | ;; + |""".stripMargin + } + + } + def nestedCommand(cmd: RegisteredCommand): String = { + val cmdStr = cmd.subcommands.map(subCmd => commandArguments(subCmd)) + val cmdName = cmd.name + + s"""_viash_$cmdName() + |{ + | case $${words[2]} in + | ${cmdStr.flatMap(s => s.split("\n")).mkString("\n| ")} + | esac + |} + |""".stripMargin + + } + + def generate(cli: CLIConf): String = { + + val (commands, nestedCommands) = cli.getRegisteredCommands(true).partition(_.subcommands.isEmpty) + + val topLevelCommandNames = cli.getRegisteredCommands().map(_.name) + + val nestedCommandsSwitch = nestedCommands.map{nc => + val ncn = nc.name + s"""$ncn) + | _viash_$ncn + | return + | ;; + |""".stripMargin + } + + val nestedCommandsSwitch2 = nestedCommands.map{nc => + val ncn = nc.name + val subcommands = nc.subcommands.map(_.name) + s"""$ncn) + | COMPREPLY=($$(compgen -W '${subcommands.mkString(" ")}' -- "$$cur")) + | return + | ;; + |""".stripMargin + } + + s"""# bash completion for viash + | + |${nestedCommands.map(nc => nestedCommand(nc)).mkString("\n")} + |_viash() + |{ + | local cur prev words cword + | _init_completion || return + | if [[ $$cword -ge 3 ]]; then + | case $${words[1]} in + | ${nestedCommandsSwitch.flatMap(_.split("\n")).mkString("\n| ")} + | esac + | fi + | + | case $$prev in + | --version | --help | -!(-*)[hV]) + | return + | ;; + | ${commands.flatMap(c => commandArguments(c).split("\n")).mkString("\n| ")} + | ${nestedCommandsSwitch2.flatMap(_.split("\n")).mkString("\n| ")} + | esac + | + | if [[ $$cur == -* ]]; then + | COMPREPLY=($$(compgen -W '-h -v' -- "$$cur")) + | elif [[ $$cword == 1 ]]; then + | COMPREPLY=($$(compgen -W '${topLevelCommandNames.mkString(" ")}' -- "$$cur")) + | fi + | + |} && + | complete -F _viash viash + |""".stripMargin + } +} + +object AutoCompleteZsh { + def commandArguments(cmd: RegisteredCommand): String = { + def removeMarkup(text: String): String = { + val markupRegex = raw"@\[(.*?)\]\(.*?\)".r + val backtickRegex = "`(\"[^`\"]*?\")`".r + val textWithoutMarkup = markupRegex.replaceAllIn(text, "$1") + backtickRegex.replaceAllIn(textWithoutMarkup, "$1") + } + def getCleanDescr(opt: RegisteredOpt): String = { + removeMarkup(opt.descr) + .replaceAll("([\\[\\]\"])", "\\\\$1") // escape square brackets and quotes + } + + val (opts, trailOpts) = cmd.opts.partition(_.optType != "trailArgs") + val cmdArgs = opts.map(o => + if (o.short.isEmpty) { + s""""--${o.name}[${getCleanDescr(o)}]"""" + } else { + s""""(-${o.short.get} --${o.name})"{-${o.short.get},--${o.name}}"[${getCleanDescr(o)}]"""" + } + ) + val cmdName = cmd.name + + trailOpts match { + case Nil => + s"""$cmdName) + | local -a cmd_args + | cmd_args=( + | ${cmdArgs.mkString("\n| ")} + | ) + | _arguments $$cmd_args $$_viash_help $$_viash_id_comp + | ;; + |""".stripMargin + case _ => + s"""$cmdName) + | if [[ $${lastParam} == -* ]]; then + | local -a cmd_args + | cmd_args=( + | ${cmdArgs.mkString("\n| ")} + | ) + | _arguments $$cmd_args $$_viash_help $$_viash_id_comp + | else + | _files + | fi + | ;; + |""".stripMargin + } + } + + + def nestedCommand(cmd: RegisteredCommand): String = { + val cmdStr = cmd.subcommands.map(subCmd => commandArguments(subCmd)) + val cmdName = cmd.name + val subCmds = cmd.subcommands.map(subCmd => s""""${subCmd.name}:${subCmd.bannerDescription.get.split("\n").head}"""") + + s"""_viash_${cmdName}_commands() { + | local -a ${cmdName}_commands + | local lastParam + | lastParam=$${words[-1]} + | + | ${cmdName}_commands=( + | ${subCmds.mkString("\n| ")} + | ) + | + | if [[ CURRENT -eq 3 ]]; then + | if [[ $${lastParam} == -* ]]; then + | _arguments $$_viash_help $$_viash_id_comp + | else + | _describe -t commands "viash subcommands" ${cmdName}_commands + | fi + | else + | case $${words[3]} in + | ${cmdStr.flatMap(s => s.split("\n")).mkString("\n| ")} + | esac + | fi + |} + |""".stripMargin + } + + def generate(cli: CLIConf) = { + + val (commands, nestedCommands) = cli.getRegisteredCommands(true).partition(_.subcommands.isEmpty) + + val topLevelCommandNames = cli.getRegisteredCommands() + + val topCmds = topLevelCommandNames.map(subCmd => s""""${subCmd.name}:${subCmd.bannerDescription.getOrElse(s"${subCmd.name} operations subcommand").split("\n").head}"""") + + val nestedCommandsSwitch = nestedCommands.map{nc => + s"""${nc.name}) + | _viash_${nc.name}_commands + | ;; + |""".stripMargin + } + + + s"""#compdef viash + | + |local -a _viash_id_comp + |_viash_id_comp=('1: :->id_comp') + | + |local -a _viash_help + |_viash_help=('(-h --help)'{-h,--help}'[Show help message]') + | + |_viash_top_commands() { + | local -a top_commands + | top_commands=( + | ${topCmds.mkString("\n| ")} + | ) + | + | _arguments \\ + | '(-v --version)'{-v,--version}'[Show verson of this program]' \\ + | $$_viash_help \\ + | $$_viash_id_comp + | + | _describe -t commands "viash subcommands" top_commands + |} + | + |${nestedCommands.map(nc => nestedCommand(nc)).mkString("\n")} + | + |_viash() { + | local lastParam + | lastParam=$${words[-1]} + | + | if [[ CURRENT -eq 2 ]]; then + | _viash_top_commands + | elif [[ CURRENT -ge 3 ]]; then + | case "$$words[2]" in + | ${commands.flatMap(c => commandArguments(c).split("\n")).mkString("\n| ")} + | ${nestedCommandsSwitch.flatMap(_.split("\n")).mkString("\n| ")} + | esac + | fi + | + | return + |} + | + |_viash + | + |# ex: filetype=sh + |""".stripMargin + } +} diff --git a/src/main/scala/io/viash/schemas/CollectedSchemas.scala b/src/main/scala/io/viash/schemas/CollectedSchemas.scala index 7c1007aef..cf0aa285e 100644 --- a/src/main/scala/io/viash/schemas/CollectedSchemas.scala +++ b/src/main/scala/io/viash/schemas/CollectedSchemas.scala @@ -33,6 +33,8 @@ import io.viash.config.Info import io.viash.functionality.resources._ import io.viash.project.ViashProject import io.viash.platforms.nextflow._ +import io.viash.helpers._ +import scala.collection.immutable.ListMap final case class CollectedSchemas ( config: Map[String, List[ParameterSchema]], @@ -72,24 +74,34 @@ object CollectedSchemas { private def getMembers[T: TypeTag](): (Map[String,List[MemberInfo]], List[Symbol]) = { - val name = typeOf[T].typeSymbol.fullName - val memberNames = typeOf[T].members - .filter(!_.isMethod) - .map(_.shortName) - .toSeq + val name = typeOf[T].typeSymbol.shortName - val constructorMembers = typeOf[T].members.filter(_.isConstructor).head.asMethod.paramLists.head.map(_.shortName) + // Get all members and filter for constructors, first one should be the best (most complete) one + // Traits don't have constructors + // Get all parameters and store their short name + val constructorMembers = typeOf[T].members.filter(_.isConstructor).headOption.map(_.asMethod.paramLists.head.map(_.shortName)).getOrElse(List.empty[String]) val baseClasses = typeOf[T].baseClasses .filter(_.fullName.startsWith("io.viash")) + // If we're only getting a abstract class/trait, not a final implementation, use these definitions (otherwise we're left with nothing). + val documentFully = + baseClasses.length == 1 && + baseClasses.head.isAbstract && + baseClasses.head.annotations.exists(a => a.tree.tpe =:= typeOf[documentFully]) + + val memberNames = typeOf[T].members + .filter(!_.isMethod || documentFully) + .map(_.shortName) + .toSeq + val allMembers = baseClasses .zipWithIndex .flatMap{ case (baseClass, index) => baseClass.info.members .filter(_.fullName.startsWith("io.viash")) .filter(m => memberNames.contains(m.shortName)) - .filter(m => !m.info.getClass.toString.endsWith("NullaryMethodType") || index != 0) // Only regular members if base class, otherwise all members + .filter(m => !m.info.getClass.toString.endsWith("NullaryMethodType") || index != 0 || documentFully) // Only regular members if base class, otherwise all members .map(y => MemberInfo(y, (constructorMembers.contains(y.shortName)), baseClass.fullName, index)) } .groupBy(k => k.shortName) @@ -97,68 +109,62 @@ object CollectedSchemas { (allMembers, baseClasses) } - val schemaClassMap = Map( - "config" -> Map( - "config" -> getMembers[Config](), - "project" -> getMembers[ViashProject](), - ), - "functionality" -> Map( - "functionality" -> getMembers[Functionality](), - "author" -> getMembers[Author](), - "computationalRequirements" -> getMembers[ComputationalRequirements](), - ), - "platforms" -> Map( - "platform" -> getMembers[Platform](), - "nativePlatform" -> getMembers[NativePlatform](), - "dockerPlatform" -> getMembers[DockerPlatform](), - "nextflowVdsl3Platform" -> getMembers[NextflowVdsl3Platform](), - "nextflowLegacyPlatform" -> getMembers[NextflowLegacyPlatform](), - ), - "requirements" -> Map( - "requirements" -> getMembers[Requirements](), - "apkRequirements" -> getMembers[ApkRequirements](), - "aptRequirements" -> getMembers[AptRequirements](), - "dockerRequirements" -> getMembers[DockerRequirements](), - "javascriptRequirements" -> getMembers[JavaScriptRequirements](), - "pythonRequirements" -> getMembers[PythonRequirements](), - "rRequirements" -> getMembers[RRequirements](), - "rubyRequirements" -> getMembers[RubyRequirements](), - "yumRequirements" -> getMembers[YumRequirements](), - ), - "arguments" -> Map( - "argument" -> getMembers[Argument[_]](), - "boolean" -> getMembers[BooleanArgument](), - "boolean_true" -> getMembers[BooleanTrueArgument](), - "boolean_false" -> getMembers[BooleanFalseArgument](), - "double" -> getMembers[DoubleArgument](), - "file" -> getMembers[FileArgument](), - "integer" -> getMembers[IntegerArgument](), - "long" -> getMembers[LongArgument](), - "string" -> getMembers[StringArgument](), - ), - "resources" -> Map( - "resource" -> getMembers[Resource](), - "bashScript" -> getMembers[BashScript](), - "cSharpScript" -> getMembers[CSharpScript](), - "executable" -> getMembers[Executable](), - "javaScriptScript" -> getMembers[JavaScriptScript](), - "nextflowScript" -> getMembers[NextflowScript](), - "plainFile" -> getMembers[PlainFile](), - "pythonScript" -> getMembers[PythonScript](), - "rScript" -> getMembers[RScript](), - "scalaScript" -> getMembers[ScalaScript](), - ), - "nextflowParameters" -> Map( - "nextflowDirectives" -> getMembers[NextflowDirectives](), - "nextflowAuto" -> getMembers[NextflowAuto](), - "nextflowConfig" -> getMembers[NextflowConfig](), - ) + lazy val schemaClasses = List( + getMembers[Config](), + getMembers[ViashProject](), + getMembers[Info](), + getMembers[SysEnvTrait](), + + getMembers[Functionality](), + getMembers[Author](), + getMembers[ComputationalRequirements](), + getMembers[ArgumentGroup](), + + getMembers[Platform](), + getMembers[NativePlatform](), + getMembers[DockerPlatform](), + getMembers[NextflowPlatform](), + + getMembers[Requirements](), + getMembers[ApkRequirements](), + getMembers[AptRequirements](), + getMembers[DockerRequirements](), + getMembers[JavaScriptRequirements](), + getMembers[PythonRequirements](), + getMembers[RRequirements](), + getMembers[RubyRequirements](), + getMembers[YumRequirements](), + + getMembers[Argument[_]](), + getMembers[BooleanArgument](), + getMembers[BooleanTrueArgument](), + getMembers[BooleanFalseArgument](), + getMembers[DoubleArgument](), + getMembers[FileArgument](), + getMembers[IntegerArgument](), + getMembers[LongArgument](), + getMembers[StringArgument](), + + getMembers[Resource](), + getMembers[BashScript](), + getMembers[CSharpScript](), + getMembers[Executable](), + getMembers[JavaScriptScript](), + getMembers[NextflowScript](), + getMembers[PlainFile](), + getMembers[PythonScript](), + getMembers[RScript](), + getMembers[ScalaScript](), + + getMembers[NextflowDirectives](), + getMembers[NextflowAuto](), + getMembers[NextflowConfig](), ) private def trimTypeName(s: String) = { // first: io.viash.helpers.data_structures.OneOrMore[String] -> OneOrMore[String] // second: List[io.viash.platforms.requirements.Requirements] -> List[Requirements] - s.replaceAll("^(\\w*\\.)*", "").replaceAll("""(\w*)\[[\w\.]*?([\w,]*)(\[_\])?\]""", "$1 of $2") + s.replaceAll("^(\\w*\\.)*", "").replaceAll("""(\w*)\[[\w\.]*?([\w,]*)(\[_\])?\]""", "$1[$2]") } private def annotationsOf(members: (Map[String,List[MemberInfo]]), classes: List[Symbol]) = { @@ -172,31 +178,23 @@ object CollectedSchemas { val annThis = ("__this__", classes.head.name.toString(), classes.head.annotations, "", 0, classes.map(_.fullName)) val allAnnotations = annThis :: annMembers.toList allAnnotations - .map({case (a, b, c, d, e, f) => (a, trimTypeName(b), f, c)}) // TODO this ignores where the annotation was defined, ie. top level class or super class + .map({case (name, tpe, annotations, d, e, hierarchy) => (name, trimTypeName(tpe), hierarchy, annotations)}) // TODO this ignores where the annotation was defined, ie. top level class or super class } private val getSchema = (t: (Map[String,List[MemberInfo]], List[Symbol])) => t match { case (members, classes) => { - annotationsOf(members, classes).flatMap{ case (a, b, c, d) => ParameterSchema(a, b, c, d) } + annotationsOf(members, classes).flatMap{ case (name, tpe, hierarchy, annotations) => ParameterSchema(name, tpe, hierarchy, annotations) } } } // Main call for documentation output - private lazy val data = CollectedSchemas( - config = schemaClassMap.get("config").get.map{ case(k, v) => (k, getSchema(v))}, - functionality = schemaClassMap.get("functionality").get.map{ case(k, v) => (k, getSchema(v))}, - platforms = schemaClassMap.get("platforms").get.map{ case(k, v) => (k, getSchema(v))}, - requirements = schemaClassMap.get("requirements").get.map{ case(k, v) => (k, getSchema(v))}, - arguments = schemaClassMap.get("arguments").get.map{ case(k, v) => (k, getSchema(v))}, - resources = schemaClassMap.get("resources").get.map{ case(k, v) => (k, getSchema(v))}, - nextflowParameters = schemaClassMap.get("nextflowParameters").get.map{ case(k, v) => (k, getSchema(v))}, - ) - - def getJson: Json = { - data.asJson - } + lazy val data: List[List[ParameterSchema]] = schemaClasses.map{ v => getSchema(v)} + + def getKeyFromParamList(data: List[ParameterSchema]): String = data.find(p => p.name == "__this__").get.`type` - private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]) = { + def getJson: Json = data.asJson + + private def getNonAnnotated(members: Map[String,List[MemberInfo]], classes: List[Symbol]): List[String] = { val issueMembers = members .toList .filter{ case(k, v) => v.map(m => m.inConstructor).contains(true) } // Only check values that are in a constructor. Annotation may occur on private vals but that is not a requirement. @@ -208,23 +206,16 @@ object CollectedSchemas { issueMembers ++ ownClassArr } + def getMemberName(members: Map[String,List[MemberInfo]], classes: List[Symbol]): String = classes.head.shortName + // Main call for checking whether all arguments are annotated - def getAllNonAnnotated = schemaClassMap.flatMap { - case (key, v1) => v1.flatMap { - case (key2, v2) => getNonAnnotated(v2._1, v2._2).map((key, key2, _)) - } - } + // Add extra non-annotated value so we can always somewhat check the code is functional + def getAllNonAnnotated: Map[String, String] = (schemaClasses :+ getMembers[CollectedSchemas]()).flatMap { + v => getNonAnnotated(v._1, v._2).map((getMemberName(v._1, v._2), _)) + }.toMap - def getAllDeprecations = { - val arr = - data.config.flatMap{ case (key, v) => v.map(v2 => ("config " + key + " " + v2.name, v2.deprecated)) } ++ - data.functionality.flatMap{ case (key, v) => v.map(v2 => ("functionality " + key + " " + v2.name, v2.deprecated)) } ++ - data.platforms.flatMap{ case (key, v) => v.map(v2 => ("platforms " + key + " " + v2.name, v2.deprecated)) } ++ - data.requirements.flatMap{ case (key, v) => v.map(v2 => ("requirements " + key + " " + v2.name, v2.deprecated)) } ++ - data.arguments.flatMap{ case (key, v) => v.map(v2 => ("arguments " + key + " " + v2.name, v2.deprecated)) } ++ - data.resources.flatMap{ case (key, v) => v.map(v2 => ("resources " + key + " " + v2.name, v2.deprecated)) } ++ - data.nextflowParameters.flatMap{ case (key, v) => v.map(v2 => ("nextflowParameters " + key + " " + v2.name, v2.deprecated)) } - + def getAllDeprecations: Map[String, DeprecatedOrRemovedSchema] = { + val arr = data.flatMap(v => v.map(p => (s"config ${getKeyFromParamList(v)} ${p.name}", p.deprecated))).toMap arr.filter(t => t._2.isDefined).map(t => (t._1, t._2.get)) } diff --git a/src/main/scala/io/viash/schemas/JsonSchema.scala b/src/main/scala/io/viash/schemas/JsonSchema.scala new file mode 100644 index 000000000..ee83bb752 --- /dev/null +++ b/src/main/scala/io/viash/schemas/JsonSchema.scala @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2020 Data Intuitive + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package io.viash.schemas + +import io.circe.Json +import io.viash.platforms.docker.DockerSetupStrategy + +object JsonSchema { + + lazy val data = CollectedSchemas.data + + def typeOrRefJson(`type`: String): (String, Json) = { + `type` match { + case "Boolean" => + "type" -> Json.fromString("boolean") + case "Int" | "Long" => + "type" -> Json.fromString("integer") + // Basic double value, still needed to compose the either (double or infinity strings) + case "Double_" => + "type" -> Json.fromString("number") + // Custom exception. Used to add infinity or nan values to the double type + case "Double" => + "$ref" -> Json.fromString("#/definitions/DoubleWithInf") + case "String" | "Path" => + "type" -> Json.fromString("string") + case "Json" => + "type" -> Json.fromString("object") + case s => + "$ref" -> Json.fromString("#/definitions/" + s) + } + } + + def valueType(`type`: String, description: Option[String] = None): Json = { + Json.obj( + description.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ + Seq(typeOrRefJson(`type`)): _* + ) + } + + def arrayType(`type`: String, description: Option[String] = None): Json = { + arrayJson(valueType(`type`), description) + } + + def mapType(`type`: String, description: Option[String] = None): Json = { + mapJson(valueType(`type`), description) + } + + def oneOrMoreType(`type`: String, description: Option[String] = None): Json = { + oneOrMoreJson(valueType(`type`, description)) + } + + def arrayJson(json: Json, description: Option[String] = None): Json = { + Json.obj( + description.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ + Seq( + "type" -> Json.fromString("array"), + "items" -> json + ): _* + ) + } + + def mapJson(json: Json, description: Option[String] = None) = { + Json.obj( + description.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil) ++ + Seq( + "type" -> Json.fromString("object"), + "additionalProperties" -> json + ): _* + ) + } + + def oneOrMoreJson(json: Json): Json = { + eitherJson( + json, + arrayJson(json) + ) + } + def eitherJson(jsons: Json*): Json = { + Json.obj("oneOf" -> Json.arr(jsons: _*)) + } + + + def getThisParameter(data: List[ParameterSchema]): ParameterSchema = + data.find(_.name == "__this__").get + + def createSchema(info: List[ParameterSchema]): (String, Json) = { + + def removeMarkup(text: String): String = { + val markupRegex = raw"@\[(.*?)\]\(.*?\)".r + val backtickRegex = "`(\"[^`\"]*?\")`".r + val textWithoutMarkup = markupRegex.replaceAllIn(text, "$1") + backtickRegex.replaceAllIn(textWithoutMarkup, "$1") + } + + val thisParameter = getThisParameter(info) + val description = removeMarkup(thisParameter.description.get) + val subclass = thisParameter.subclass.map(l => l.head) + val properties = info.filter(p => !p.name.startsWith("__")).filter(p => !p.removed.isDefined) + val propertiesJson = properties.map(p => { + val pDescription = p.description.map(s => removeMarkup(s)) + val trimmedType = p.`type` match { + case s if s.startsWith("Option[") => s.stripPrefix("Option[").stripSuffix("]") + case s => s + } + + val mapRegex = "(List)?Map\\[String,(\\w*)\\]".r + + trimmedType match { + case s"List[$s]" => + (p.name, arrayType(s, pDescription)) + + case "Either[String,List[String]]" => + (p.name, eitherJson( + valueType("String", pDescription), + arrayType("String", pDescription) + )) + + case "Either[Map[String,String],String]" => + (p.name, eitherJson( + mapType("String", pDescription), + valueType("String", pDescription) + )) + + case s"Either[$s,$t]" => + (p.name, eitherJson( + valueType(s, pDescription), + valueType(t, pDescription) + )) + + case "OneOrMore[Map[String,String]]" => + (p.name, oneOrMoreJson( + mapType("String", pDescription) + )) + + case "OneOrMore[Either[String,Map[String,String]]]" => + (p.name, oneOrMoreJson( + eitherJson( + valueType("String", pDescription), + mapType("String", pDescription) + ) + )) + + case s"OneOrMore[$s]" => + if (s == "String" && p.name == "port" && subclass == Some("docker")) { + // Custom exception + // This is the port field for a docker platform. + // We want to allow a Strings or Ints. + (p.name, eitherJson( + valueType("String", pDescription), + valueType("Int", pDescription), + arrayType("String", pDescription), + arrayType("Int", pDescription) + )) + } else { + (p.name, oneOrMoreType(s, pDescription)) + } + + case mapRegex(_, s) => + (p.name, mapType(s, pDescription)) + + case s if p.name == "type" && subclass.isDefined => + ("type", Json.obj( + "description" -> Json.fromString(description), // not pDescription! We want to show the description of the main class + "const" -> Json.fromString(subclass.get) + )) + + case s => + (p.name, valueType(s, pDescription)) + } + + }) + + val required = properties.filter(p => + !( + p.`type`.startsWith("Option[") || + p.default.isDefined || + (p.name == "type" && thisParameter.`type` == "PlainFile") // Custom exception, file resources are "kind of" default + )) + val requiredJson = required.map(p => Json.fromString(p.name)) + + val k = thisParameter.`type` + val v = Json.obj( + "description" -> Json.fromString(description), + "type" -> Json.fromString("object"), + "properties" -> Json.obj(propertiesJson: _*), + "required" -> Json.arr(requiredJson: _*), + "additionalProperties" -> Json.False + ) + k -> v + } + + def createSuperClassSchema(info: List[ParameterSchema]): (String, Json) = { + val thisParameter = getThisParameter(info) + val k = thisParameter.`type` + val v = eitherJson( + thisParameter.subclass.get.map(s => Json.obj("$ref" -> Json.fromString(s"#/definitions/$s"))): _* + ) + k -> v + } + + def createSchemas(data: List[List[ParameterSchema]]) : Seq[(String, Json)] = { + data.flatMap{ + case v if getThisParameter(v).removed.isDefined => None + case v if getThisParameter(v).subclass.map(_.length).getOrElse(0) > 1 => Some(createSuperClassSchema(v)) + case v => Some(createSchema(v)) + } + } + + def createEnum(values: Seq[String], description: Option[String], comment: Option[String]): Json = { + Json.obj( + Seq("enum" -> Json.arr(values.map(s => Json.fromString(s)): _*)) ++ + comment.map(s => Seq("$comment" -> Json.fromString(s))).getOrElse(Nil) ++ + description.map(s => Seq("description" -> Json.fromString(s))).getOrElse(Nil): _* + ) + } + + def getJsonSchema: Json = { + val definitions = + createSchemas(data) ++ + Seq( + "DockerSetupStrategy" -> createEnum(DockerSetupStrategy.map.keys.toSeq, Some("The Docker setup strategy to use when building a container."), Some("TODO add descriptions to different strategies")), + "Direction" -> createEnum(Seq("input", "output"), Some("Makes this argument an `input` or an `output`, as in does the file/folder needs to be read or written. `input` by default."), None), + "Status" -> createEnum(Seq("enabled", "disabled", "deprecated"), Some("Allows setting a component to active, deprecated or disabled."), None), + "DockerResolveVolume" -> createEnum(Seq("manual", "automatic", "auto", "Manual", "Automatic", "Auto"), Some("Enables or disables automatic volume mapping. Enabled when set to `Automatic` or disabled when set to `Manual`. Default: `Automatic`"), Some("TODO make fully case insensitive")), + "DoubleStrings" -> createEnum(Seq("+.inf", "+inf", "+infinity", "positiveinfinity", "positiveinf", "-.inf", "-inf", "-infinity", "negativeinfinity", "negativeinf", ".nan", "nan"), None, None) + ) ++ + Seq("DoubleWithInf" -> eitherJson(valueType("Double_"), valueType("DoubleStrings"))) + + Json.obj( + "$schema" -> Json.fromString("https://json-schema.org/draft-07/schema#"), + "definitions" -> Json.obj( + definitions: _* + ), + "oneOf" -> Json.arr(valueType("Config")) + ) + } +} diff --git a/src/main/scala/io/viash/schemas/ParameterSchema.scala b/src/main/scala/io/viash/schemas/ParameterSchema.scala index 822baf586..3a1dbf398 100644 --- a/src/main/scala/io/viash/schemas/ParameterSchema.scala +++ b/src/main/scala/io/viash/schemas/ParameterSchema.scala @@ -22,12 +22,15 @@ import scala.reflect.runtime.universe._ final case class ParameterSchema( name: String, `type`: String, + niceType: String, hierarchy: Option[List[String]], description: Option[String], example: Option[List[ExampleSchema]], since: Option[String], deprecated: Option[DeprecatedOrRemovedSchema], removed: Option[DeprecatedOrRemovedSchema], + default: Option[String], + subclass: Option[List[String]], ) object ParameterSchema { @@ -73,15 +76,68 @@ object ParameterSchema { } def apply(name: String, `type`: String, hierarchy: List[String], annotations: List[Annotation]): Option[ParameterSchema] = { - // name is e.g. "io.viash.functionality.Functionality.name", only keep "name" - // name can also be "__this__" - val name_ = name.split('.').last + + def beautifyTypeName(s: String): String = { + + // "tpe[a]" -> "(\w*)\[(\w*)\]" + def regexify(s: String) = s.replace("[", "\\[").replace("]", "\\]").replaceAll("\\w+", "(\\\\w+)").r + + val regex0 = regexify("tpe") + val regex1 = regexify("tpe[a]") + val regex2 = regexify("tpe[a,b]") + val regexNested1 = regexify("tpe[a[b,c]]") + val regexNested2 = regexify("tpe[a[b,c[d,e]]]") + val regexNested3 = regexify("tpe[a,b[c,d]]") + val regexNested4 = regexify("tpe[a[b[c,d],e]]") + val regexNested5 = regexify("tpe[a[b,c[d]]]") + val regexNested6 = regexify("tpe[a[b,c],d]") + val regexNested7 = regexify("tpe[a,b[c]]") + + def map(a: String, b: String): String = s"Map of $a to $b" + def either(a: String, b: String): String = + s"""Either + | - $a + | - $b""".stripMargin + + s match { + case regex0(tpe) => s"$tpe" + case regex1(tpe, subtpe) => s"$tpe of $subtpe" + case regex2("Map", subtpe1, subtpe2) => map(subtpe1,subtpe2) + case regex2("ListMap", subtpe1, subtpe2) => map(subtpe1, subtpe2) + case regex2("Either", subtpe1, subtpe2) => either(subtpe1, subtpe2) + case regexNested1(tpe, a, b, c) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c]")}" + case regexNested2(tpe, a, b ,c ,d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d,$e]]")}" + case regexNested3("Either", a, b, c, d) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c,$d]")) + case regexNested4(tpe, a, b, c, d, e) => s"$tpe of ${beautifyTypeName(s"$a[$b[$c,$d],$e]")}" + case regexNested5(tpe, a, b, c, d) => s"$tpe of ${beautifyTypeName(s"$a[$b,$c[$d]]")}" + case regexNested6("Either", a, b, c, d) => either(beautifyTypeName(s"$a[$b,$c]"), beautifyTypeName(d)) + case regexNested7("Either", a, b, c) => either(beautifyTypeName(a), beautifyTypeName(s"$b[$c]")) + case _ => s + } + } + val annStrings = annotations.map(annotationToStrings(_)) val hierarchyOption = hierarchy match { case l if l.length > 0 => Some(l) case _ => None } + // name is e.g. "io.viash.functionality.Functionality.name", only keep "name" + // name can also be "__this__" + // Use the name defined from the class, *unless* the 'nameOverride' annotation is set. Then use the override, unless the name is '__this__'. + val nameOverride = annStrings.collectFirst({case (name, value) if name.endsWith("nameOverride") => value.head}) + val nameFromClass = name.split('.').last + val name_ = (nameOverride, nameFromClass) match { + case (Some(_), "__this__") => "__this__" + case (Some(ann), _) => ann + case (None, name) => name + } + + val typeName = (`type`, nameOverride, nameFromClass) match { + case (_, Some(newTypeName), "__this__") => newTypeName + case (typeName, _, _) => typeName + } + val description = annStrings.collectFirst({case (name, value) if name.endsWith("description") => value.head}) val example = annStrings.collect({case (name, value) if name.endsWith("example") => value}).map(ExampleSchema(_)) val exampleWithDescription = annStrings.collect({case (name, value) if name.endsWith("exampleWithDescription") => value}).map(ExampleSchema(_)) @@ -92,12 +148,19 @@ object ParameterSchema { val since = annStrings.collectFirst({case (name, value) if name.endsWith("since") => value.head}) val deprecated = annStrings.collectFirst({case (name, value) if name.endsWith("deprecated") => value}).map(DeprecatedOrRemovedSchema(_)) val removed = annStrings.collectFirst({case (name, value) if name.endsWith("removed") => value}).map(DeprecatedOrRemovedSchema(_)) + val defaultFromAnnotation = annStrings.collectFirst({case (name, value) if name.endsWith("default") => value.head}) + val defaultFromType = Option.when(typeName.startsWith("Option["))("Empty") + val default = defaultFromAnnotation orElse defaultFromType + val subclass = annStrings.collect{ case (name, value) if name.endsWith("subclass") => value.head } match { + case l if l.nonEmpty => Some(l) + case _ => None + } val undocumented = annStrings.exists{ case (name, value) => name.endsWith("undocumented")} val internalFunctionality = annStrings.exists{ case (name, value) => name.endsWith("internalFunctionality")} internalFunctionality || undocumented match { case true => None - case _ => Some(ParameterSchema(name_, `type`, hierarchyOption, description, examples, since, deprecated, removed)) + case _ => Some(ParameterSchema(name_, typeName, beautifyTypeName(typeName), hierarchyOption, description, examples, since, deprecated, removed, default, subclass)) } } diff --git a/src/main/scala/io/viash/schemas/package.scala b/src/main/scala/io/viash/schemas/package.scala index 2db7901b3..87469aaaf 100644 --- a/src/main/scala/io/viash/schemas/package.scala +++ b/src/main/scala/io/viash/schemas/package.scala @@ -38,9 +38,24 @@ package object schemas { @getter @setter @beanGetter @beanSetter @field class removed(message: String, deprecatedSince: String, since: String) extends scala.annotation.StaticAnnotation + @getter @setter @beanGetter @beanSetter @field + class default(default: String) extends scala.annotation.StaticAnnotation + @getter @setter @beanGetter @beanSetter @field class internalFunctionality() extends scala.annotation.StaticAnnotation @getter @setter @beanGetter @beanSetter @field class undocumented() extends scala.annotation.StaticAnnotation + + // In case of abstract classes; don't filter members + @getter @setter @beanGetter @beanSetter @field + class documentFully() extends scala.annotation.StaticAnnotation + + @getter @setter @beanGetter @beanSetter @field + class nameOverride(name: String) extends scala.annotation.StaticAnnotation + + // Used in either child classes or the super class. + // If used in the child class then use the yaml value, or used in the super class and then use the class name + @getter @setter @beanGetter @beanSetter @field + class subclass(name: String) extends scala.annotation.StaticAnnotation } diff --git a/src/test/resources/test_languages/bash/config.vsh.yaml b/src/test/resources/test_languages/bash/config.vsh.yaml index 5059f1b66..aaee18788 100755 --- a/src/test/resources/test_languages/bash/config.vsh.yaml +++ b/src/test/resources/test_languages/bash/config.vsh.yaml @@ -1,5 +1,6 @@ functionality: __merge__: [., ../common.yaml] + name: test_languages_bash resources: - type: bash_script path: ./code.sh diff --git a/src/test/resources/test_languages/common.yaml b/src/test/resources/test_languages/common.yaml index 0dbdc8cd7..f232c7b00 100644 --- a/src/test/resources/test_languages/common.yaml +++ b/src/test/resources/test_languages/common.yaml @@ -1,4 +1,3 @@ -name: test_languages description: | Prints out the parameter values. Checking what happens with multiline descriptions. diff --git a/src/test/resources/test_languages/csharp/config.vsh.yaml b/src/test/resources/test_languages/csharp/config.vsh.yaml index 16cef708b..d68e36608 100644 --- a/src/test/resources/test_languages/csharp/config.vsh.yaml +++ b/src/test/resources/test_languages/csharp/config.vsh.yaml @@ -1,5 +1,6 @@ functionality: __merge__: [., ../common.yaml] + name: test_languages_csharp resources: - type: csharp_script path: script.csx diff --git a/src/test/resources/test_languages/csharp/script.csx b/src/test/resources/test_languages/csharp/script.csx index 027eeea4f..c1e513ac1 100644 --- a/src/test/resources/test_languages/csharp/script.csx +++ b/src/test/resources/test_languages/csharp/script.csx @@ -69,11 +69,38 @@ try { if (p.PropertyType.IsArray) { - object[] array = (object[])p.GetValue(par); + var array = p.GetValue(par) as Array; + if (array.Length == 0) Output($"{p.Name}: |empty array|"); - else + else if (array is bool[]) + { + var array2 = (array as bool[]).Select(x => x.ToString().ToLower()); + Output($"{p.Name}: |{string.Join(":", array2)}|"); + } + else if (array is System.Int32[]) + { + var array2 = array as System.Int32[]; + Output($"{p.Name}: |{string.Join(":", array2)}|"); + } + else if (array is System.Int64[]) + { + var array2 = array as System.Int64[]; + Output($"{p.Name}: |{string.Join(":", array2)}|"); + } + else if (array is System.Double[]) + { + var array2 = array as System.Double[]; + Output($"{p.Name}: |{string.Join(":", array2)}|"); + } + else if (array is System.String[]) + { + var array2 = array as System.String[]; + Output($"{p.Name}: |{string.Join(":", array2)}|"); + } + else { Output($"{p.Name}: |{string.Join(":", array)}|"); + } } else { diff --git a/src/test/resources/test_languages/js/config.vsh.yaml b/src/test/resources/test_languages/js/config.vsh.yaml index 1ed032e5c..b35a5f88f 100644 --- a/src/test/resources/test_languages/js/config.vsh.yaml +++ b/src/test/resources/test_languages/js/config.vsh.yaml @@ -1,5 +1,6 @@ functionality: __merge__: [., ../common.yaml] + name: test_languages_js resources: - type: javascript_script path: ./code.js diff --git a/src/test/resources/test_languages/multi-boolean.sh b/src/test/resources/test_languages/multi-boolean.sh new file mode 100755 index 000000000..cadbe2cd8 --- /dev/null +++ b/src/test/resources/test_languages/multi-boolean.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ex + +echo ">>> Checking whether expected resources exist" +[[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 +[[ ! -f "$meta_resources_dir/.config.vsh.yaml" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 +[[ ! -f "$meta_config" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 + +echo ">>> Checking whether output is correct" +"$meta_executable" "resource1.txt" --real_number 10.5 --whole_number=10 -s "a string with spaces" \ + true true false true \ + --output ./output.txt --log ./log.txt \ + --multiple true --multiple=false \ + false false \ + --long_number 112589990684262400 + +[[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 +grep -q 'input: |resource1.txt|' output.txt +grep -q 'real_number: |10.5|' output.txt +grep -q 'whole_number: |10|' output.txt +grep -q 'long_number: |112589990684262400|' output.txt +grep -q 's: |a string with spaces|' output.txt +grep -q 'multiple: |true:false|' output.txt +grep -q 'multiple_pos: |true:true:false:true:false:false|' output.txt +echo ">>> Test finished successfully" diff --git a/src/test/resources/test_languages/multi-double.sh b/src/test/resources/test_languages/multi-double.sh new file mode 100755 index 000000000..a829855c7 --- /dev/null +++ b/src/test/resources/test_languages/multi-double.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ex + +echo ">>> Checking whether expected resources exist" +[[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 +[[ ! -f "$meta_resources_dir/.config.vsh.yaml" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 +[[ ! -f "$meta_config" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 + +echo ">>> Checking whether output is correct" +"$meta_executable" "resource1.txt" --real_number 10.5 --whole_number=10 -s "a string with spaces" \ + 1.1 2.2 3.3 4.4 \ + --output ./output.txt --log ./log.txt \ + --multiple 5.5 --multiple=38.1 \ + 123.123 456.456 \ + --long_number 112589990684262400 + +[[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 +grep -q 'input: |resource1.txt|' output.txt +grep -q 'real_number: |10.5|' output.txt +grep -q 'whole_number: |10|' output.txt +grep -q 'long_number: |112589990684262400|' output.txt +grep -q 's: |a string with spaces|' output.txt +grep -q 'multiple: |5.5:38.1|' output.txt +grep -q 'multiple_pos: |1.1:2.2:3.3:4.4:123.123:456.456|' output.txt +echo ">>> Test finished successfully" diff --git a/src/test/resources/test_languages/multi-file.sh b/src/test/resources/test_languages/multi-file.sh new file mode 100755 index 000000000..566b7cc46 --- /dev/null +++ b/src/test/resources/test_languages/multi-file.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ex + +echo ">>> Checking whether expected resources exist" +[[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 +[[ ! -f "$meta_resources_dir/.config.vsh.yaml" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 +[[ ! -f "$meta_config" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 + +echo ">>> Checking whether output is correct" +"$meta_executable" "resource1.txt" --real_number 10.5 --whole_number=10 -s "a string with spaces" \ + a.sh b.sh c.sh d.sh \ + --output ./output.txt --log ./log.txt \ + --multiple abc.txt --multiple=def.txt \ + e.sh f.sh \ + --long_number 112589990684262400 + +[[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 +grep -q 'input: |resource1.txt|' output.txt +grep -q 'real_number: |10.5|' output.txt +grep -q 'whole_number: |10|' output.txt +grep -q 'long_number: |112589990684262400|' output.txt +grep -q 's: |a string with spaces|' output.txt +grep -q 'multiple: |abc.txt:def.txt|' output.txt +grep -q 'multiple_pos: |a.sh:b.sh:c.sh:d.sh:e.sh:f.sh|' output.txt +echo ">>> Test finished successfully" diff --git a/src/test/resources/test_languages/multi-integer.sh b/src/test/resources/test_languages/multi-integer.sh new file mode 100755 index 000000000..8bc475791 --- /dev/null +++ b/src/test/resources/test_languages/multi-integer.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ex + +echo ">>> Checking whether expected resources exist" +[[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 +[[ ! -f "$meta_resources_dir/.config.vsh.yaml" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 +[[ ! -f "$meta_config" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 + +echo ">>> Checking whether output is correct" +"$meta_executable" "resource1.txt" --real_number 10.5 --whole_number=10 -s "a string with spaces" \ + 1 2 3 4 \ + --output ./output.txt --log ./log.txt \ + --multiple 5 --multiple=38 \ + 123 456 \ + --long_number 112589990684262400 + +[[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 +grep -q 'input: |resource1.txt|' output.txt +grep -q 'real_number: |10.5|' output.txt +grep -q 'whole_number: |10|' output.txt +grep -q 'long_number: |112589990684262400|' output.txt +grep -q 's: |a string with spaces|' output.txt +grep -q 'multiple: |5:38|' output.txt +grep -q 'multiple_pos: |1:2:3:4:123:456|' output.txt +echo ">>> Test finished successfully" diff --git a/src/test/resources/test_languages/multi-long.sh b/src/test/resources/test_languages/multi-long.sh new file mode 100755 index 000000000..6d353374b --- /dev/null +++ b/src/test/resources/test_languages/multi-long.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -ex + +echo ">>> Checking whether expected resources exist" +[[ ! -f "$meta_executable" ]] && echo "executable could not be found!" && exit 1 +[[ ! -f "$meta_resources_dir/.config.vsh.yaml" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 +[[ ! -f "$meta_config" ]] && echo ".config.vsh.yaml could not be found!" && exit 1 + +echo ">>> Checking whether output is correct" +"$meta_executable" "resource1.txt" --real_number 10.5 --whole_number=10 -s "a string with spaces" \ + 446913741939 338239080089 864531271886 126957339937 \ + --output ./output.txt --log ./log.txt \ + --multiple 806082089013 --multiple=360278033202 \ + 829243285718 694515245636 \ + --long_number 112589990684262400 + +[[ ! -f output.txt ]] && echo "Output file could not be found!" && exit 1 +grep -q 'input: |resource1.txt|' output.txt +grep -q 'real_number: |10.5|' output.txt +grep -q 'whole_number: |10|' output.txt +grep -q 'long_number: |112589990684262400|' output.txt +grep -q 's: |a string with spaces|' output.txt +grep -q 'multiple: |806082089013:360278033202|' output.txt +grep -q 'multiple_pos: |446913741939:338239080089:864531271886:126957339937:829243285718:694515245636|' output.txt +echo ">>> Test finished successfully" diff --git a/src/test/resources/test_languages/python/code.py b/src/test/resources/test_languages/python/code.py index ffcc2be11..9a4cfad8a 100644 --- a/src/test/resources/test_languages/python/code.py +++ b/src/test/resources/test_languages/python/code.py @@ -44,7 +44,11 @@ def echo(s): if not value: echo(f"{key}: |empty array|") else: - echo(f"{key}: |{':'.join(value)}|") + if isinstance(value[0] , bool): + value2 = map(lambda v: str(v).lower(), value) + else: + value2 = map(lambda v: str(v), value) + echo(f"{key}: |{':'.join(value2)}|") elif value is None: echo(f"{key}: ||") elif isinstance(value, bool): diff --git a/src/test/resources/test_languages/python/config.vsh.yaml b/src/test/resources/test_languages/python/config.vsh.yaml index 3556db7b5..8da7927b2 100644 --- a/src/test/resources/test_languages/python/config.vsh.yaml +++ b/src/test/resources/test_languages/python/config.vsh.yaml @@ -1,5 +1,6 @@ functionality: __merge__: [., ../common.yaml] + name: test_languages_python resources: - type: python_script path: code.py diff --git a/src/test/resources/test_languages/r/script.vsh.R b/src/test/resources/test_languages/r/script.vsh.R index 4119293a4..57e5bfe97 100644 --- a/src/test/resources/test_languages/r/script.vsh.R +++ b/src/test/resources/test_languages/r/script.vsh.R @@ -1,5 +1,6 @@ #' functionality: #' __merge__: [., ../common.yaml] +#' name: test_languages_r #' platforms: #' - type: native #' - type: docker diff --git a/src/test/resources/test_languages/scala/config.vsh.yaml b/src/test/resources/test_languages/scala/config.vsh.yaml index a39af2127..1bf0c1a3d 100644 --- a/src/test/resources/test_languages/scala/config.vsh.yaml +++ b/src/test/resources/test_languages/scala/config.vsh.yaml @@ -1,5 +1,6 @@ functionality: __merge__: [., ../common.yaml] + name: test_languages_scala resources: - type: scala_script path: script.scala diff --git a/src/test/resources/test_languages/test.sh b/src/test/resources/test_languages/test.sh index 7bc015fd7..fc9ddb2cc 100755 --- a/src/test/resources/test_languages/test.sh +++ b/src/test/resources/test_languages/test.sh @@ -31,7 +31,7 @@ grep -q 'optional: |foo|' output.txt grep -q 'optional_with_default: |bar|' output.txt grep -q 'multiple: |one:two|' output.txt grep -q 'multiple_pos: |a:b:c:d:e:f|' output.txt -grep -q 'meta_functionality_name: |test_languages|' output.txt +grep -q 'meta_functionality_name: |test_languages_.*|' output.txt grep -q 'meta_resources_dir: |..*|' output.txt grep -q 'meta_cpus: |2|' output.txt grep -q 'meta_memory_b: |2147483648|' output.txt @@ -73,7 +73,7 @@ grep -q 'optional_with_default: |The default value.|' output2.txt grep -q 'multiple: ||' output2.txt grep -q 'multiple_pos: ||' output2.txt -grep -q 'meta_functionality_name: |test_languages|' output2.txt +grep -q 'meta_functionality_name: |test_languages_.*|' output2.txt grep -q 'meta_resources_dir: |..*|' output2.txt grep -q 'meta_cpus: |666|' output2.txt grep -q 'meta_memory_b: |112589990684262400|' output2.txt diff --git a/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml b/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml index add5dea57..f429e5986 100755 --- a/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml +++ b/src/test/resources/testbash/auxiliary_requirements/config_requirements.vsh.yaml @@ -23,33 +23,3 @@ functionality: platforms: - type: docker image: "bash:3.2" - id: "viash_requirement_apk_base" - - type: docker - image: "bash:3.2" - target_image: "viash_requirement_apk" - id: "viash_requirement_apk" - setup: - - type: apk - packages: - - fortune - - type: docker - image: debian:bullseye-slim - target_image: "viash_requirement_apt_base" - id: "viash_requirement_apt_base" - - type: docker - image: debian:bullseye-slim - target_image: "viash_requirement_apt" - id: "viash_requirement_apt" - setup: - - type: apt - packages: - - cowsay - - type: docker - image: "bash:3.2" - target_image: "viash_requirement_apk" - id: "viash_requirement_apk_test_setup" - test_setup: - - type: apk - packages: - - fortune - diff --git a/src/test/resources/testbash/auxiliary_resource/config_resource_unsupported_protocol.vsh.yaml b/src/test/resources/testbash/auxiliary_resource/config_resource_unsupported_protocol.vsh.yaml deleted file mode 100755 index 11d67d6e7..000000000 --- a/src/test/resources/testbash/auxiliary_resource/config_resource_unsupported_protocol.vsh.yaml +++ /dev/null @@ -1,21 +0,0 @@ -functionality: - name: testbash - description: | - Test various ways of specifying resources and check they ended up being in the right place. - resources: - - type: bash_script - path: ./check_bash_version.sh - - type: bash_script - path: ./code.sh - - path: resource1.txt - - path: ./resource2.txt - - path: ftp://ftp.ubuntu.com/releases/robots.txt - - arguments: - - name: "--optional" - type: string - description: An optional string. -platforms: - - type: native - - type: docker - image: "bash:3.2" diff --git a/src/test/resources/testbash/config_no_platform.vsh.yaml b/src/test/resources/testbash/config_no_platform.vsh.yaml deleted file mode 100755 index e8f765acd..000000000 --- a/src/test/resources/testbash/config_no_platform.vsh.yaml +++ /dev/null @@ -1,68 +0,0 @@ -functionality: - name: testbash - description: | - Prints out the parameter values. - Checking what happens with multiline descriptions. - arguments: - - name: "input" - type: file - description: | - An input file with positional arguments. - More checks for multiline descriptions. - Testing some characters that should be escaped: ` $ \ - direction: input - required: true - must_exist: true - - name: "--real_number" - type: double - description: A real number with positional arguments. - required: true - - name: "--whole_number" - type: integer - description: A whole number with a standard flag. - required: true - - name: "-s" - type: string - description: A sentence or word with a short flag. - required: true - - name: "--truth" - type: boolean_true - description: A switch flag. - - name: "--falsehood" - type: boolean_false - description: A switch flag which is false when specified. - - name: "--reality" - type: boolean - description: A switch flag without predetermined state. - - name: "--output" - alternatives: [ "-o" ] - type: file - description: Write the parameters to a json file. - direction: output - - name: "--log" - type: file - description: An optional log file. - direction: output - - name: "--optional" - type: string - description: An optional string. - - name: "--optional_with_default" - type: string - default: "The default value." - - name: "--multiple" - type: string - multiple: true - - name: "multiple_pos" - type: string - multiple: true - resources: - - type: bash_script - path: ./code.sh - - path: resource1.txt - - path: https://raw.githubusercontent.com/scala/scala/fff4ec3539ac58f56fdc8f1382c365f32a9fd25a/NOTICE - test_resources: - - type: bash_script - path: tests/check_outputs.sh - - type: bash_script - path: tests/fail.sh - - path: resource2.txt diff --git a/src/test/resources/testbash/config_nonexistent_test.vsh.yaml b/src/test/resources/testbash/config_nonexistent_test.vsh.yaml deleted file mode 100755 index afa01edfe..000000000 --- a/src/test/resources/testbash/config_nonexistent_test.vsh.yaml +++ /dev/null @@ -1,64 +0,0 @@ -functionality: - name: testbash - description: | - Prints out the parameter values. - Checking what happens with multiline descriptions. - arguments: - - name: "input" - type: file - description: | - An input file with positional arguments. - More checks for multiline descriptions. - Testing some characters that should be escaped: ` $ \ - direction: input - required: true - must_exist: true - - name: "--real_number" - type: double - description: A real number with positional arguments. - required: true - - name: "--whole_number" - type: integer - description: A whole number with a standard flag. - required: true - - name: "-s" - type: string - description: A sentence or word with a short flag. - required: true - - name: "--truth" - type: boolean_true - description: A switch flag. - - name: "--output" - alternatives: [ "-o" ] - type: file - description: Write the parameters to a json file. - direction: output - - name: "--log" - type: file - description: An optional log file. - direction: output - - name: "--optional" - type: string - description: An optional string. - - name: "--optional_with_default" - type: string - default: "The default value." - - name: "--multiple" - type: string - multiple: true - - name: "multiple_pos" - type: string - multiple: true - resources: - - type: bash_script - path: ./code.sh - - path: resource1.txt - - path: https://raw.githubusercontent.com/scala/scala/fff4ec3539ac58f56fdc8f1382c365f32a9fd25a/NOTICE - test_resources: - - type: bash_script - path: tests/nonexistent_test.sh -platforms: - - type: native - - type: docker - image: "bash:3.2" - - type: nextflow diff --git a/src/test/resources/testbash/docker_options/config_chown.vsh.yaml b/src/test/resources/testbash/docker_options/config_chown.vsh.yaml deleted file mode 100755 index 867e48542..000000000 --- a/src/test/resources/testbash/docker_options/config_chown.vsh.yaml +++ /dev/null @@ -1,74 +0,0 @@ -functionality: - name: testbash - description: | - Prints out the parameter values. - Checking what happens with multiline descriptions. - arguments: - - name: "input" - type: file - description: | - An input file with positional arguments. - More checks for multiline descriptions. - Testing some characters that should be escaped: ` $ \ - direction: input - required: true - must_exist: true - - name: "--real_number" - type: double - description: A real number with positional arguments. - required: true - - name: "--whole_number" - type: integer - description: A whole number with a standard flag. - required: true - - name: "-s" - type: string - description: A sentence or word with a short flag. - required: true - - name: "--truth" - type: boolean_true - description: A switch flag. - - name: "--falsehood" - type: boolean_false - description: A switch flag which is false when specified. - - name: "--reality" - type: boolean - description: A switch flag without predetermined state. - - name: "--output" - alternatives: [ "-o" ] - type: file - description: Write the parameters to a json file. - direction: output - - name: "--log" - type: file - description: An optional log file. - direction: output - - name: "--optional" - type: string - description: An optional string. - - name: "--optional_with_default" - type: string - default: "The default value." - - name: "--multiple" - type: string - multiple: true - - name: "multiple_pos" - type: string - multiple: true - resources: - - type: bash_script - path: ./code.sh - - path: ../resource1.txt - - path: https://raw.githubusercontent.com/scala/scala/fff4ec3539ac58f56fdc8f1382c365f32a9fd25a/NOTICE -platforms: - - type: docker - image: "bash:3.2" - id: "chown_default" - - type: docker - image: "bash:3.2" - id: "chown_true" - chown: true - - type: docker - image: "bash:3.2" - id: "chown_false" - chown: false diff --git a/src/test/resources/testbash/docker_options/config_chown_multiple_output.vsh.yaml b/src/test/resources/testbash/docker_options/config_chown_multiple_output.vsh.yaml deleted file mode 100755 index 34ef216c6..000000000 --- a/src/test/resources/testbash/docker_options/config_chown_multiple_output.vsh.yaml +++ /dev/null @@ -1,73 +0,0 @@ -functionality: - name: testbash - description: | - Prints out the parameter values. - Checking what happens with multiline descriptions. - arguments: - - name: "input" - type: file - description: | - An input file with positional arguments. - More checks for multiline descriptions. - Testing some characters that should be escaped: ` $ \ - direction: input - required: true - must_exist: true - - name: "--real_number" - type: double - description: A real number with positional arguments. - required: true - - name: "--whole_number" - type: integer - description: A whole number with a standard flag. - required: true - - name: "-s" - type: string - description: A sentence or word with a short flag. - required: true - - name: "--truth" - type: boolean_true - description: A switch flag. - - name: "--falsehood" - type: boolean_false - description: A switch flag which is false when specified. - - name: "--reality" - type: boolean - description: A switch flag without predetermined state. - - name: "--output" - alternatives: [ "-o" ] - type: file - description: Write the parameters to a json file. - direction: output - multiple: true - - name: "--log" - type: file - description: An optional log file. - direction: output - - name: "--optional" - type: string - description: An optional string. - - name: "--optional_with_default" - type: string - default: "The default value." - - name: "output_pos" - type: file - direction: output - multiple: true - resources: - - type: bash_script - path: ./code_multiple_output.sh - - path: ../resource1.txt - - path: https://raw.githubusercontent.com/scala/scala/fff4ec3539ac58f56fdc8f1382c365f32a9fd25a/NOTICE -platforms: - - type: docker - image: "bash:3.2" - id: "multiple_chown_default" - - type: docker - image: "bash:3.2" - id: "multiple_chown_true" - chown: true - - type: docker - image: "bash:3.2" - id: "multiple_chown_false" - chown: false diff --git a/src/test/resources/testbash/docker_options/config_chown_two_output.vsh.yaml b/src/test/resources/testbash/docker_options/config_chown_two_output.vsh.yaml deleted file mode 100755 index 7dce73e26..000000000 --- a/src/test/resources/testbash/docker_options/config_chown_two_output.vsh.yaml +++ /dev/null @@ -1,78 +0,0 @@ -functionality: - name: testbash - description: | - Prints out the parameter values. - Checking what happens with multiline descriptions. - arguments: - - name: "input" - type: file - description: | - An input file with positional arguments. - More checks for multiline descriptions. - Testing some characters that should be escaped: ` $ \ - direction: input - required: true - must_exist: true - - name: "--real_number" - type: double - description: A real number with positional arguments. - required: true - - name: "--whole_number" - type: integer - description: A whole number with a standard flag. - required: true - - name: "-s" - type: string - description: A sentence or word with a short flag. - required: true - - name: "--truth" - type: boolean_true - description: A switch flag. - - name: "--falsehood" - type: boolean_false - description: A switch flag which is false when specified. - - name: "--reality" - type: boolean - description: A switch flag without predetermined state. - - name: "--output" - alternatives: [ "-o" ] - type: file - description: Write the parameters to a json file. - direction: output - - name: "--output2" - type: file - description: Write the parameters to a json file. - direction: output - - name: "--log" - type: file - description: An optional log file. - direction: output - - name: "--optional" - type: string - description: An optional string. - - name: "--optional_with_default" - type: string - default: "The default value." - - name: "--multiple" - type: string - multiple: true - - name: "multiple_pos" - type: string - multiple: true - resources: - - type: bash_script - path: ./code_two_output.sh - - path: ../resource1.txt - - path: https://raw.githubusercontent.com/scala/scala/fff4ec3539ac58f56fdc8f1382c365f32a9fd25a/NOTICE -platforms: - - type: docker - image: "bash:3.2" - id: "two_chown_default" - - type: docker - image: "bash:3.2" - id: "two_chown_true" - chown: true - - type: docker - image: "bash:3.2" - id: "two_chown_false" - chown: false diff --git a/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf b/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf index c38d84e25..434056f37 100644 --- a/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf +++ b/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf @@ -11,9 +11,6 @@ include { step2 } from "$targetDir/step2/main.nf" // ["input": List[File]] -> File include { step3 } from "$targetDir/step3/main.nf" -params.displayDebug = false // Default value, still settable by command line -def debug = params.displayDebug - def channelValue = Channel.value([ "foo", // id [ // data @@ -25,7 +22,7 @@ def channelValue = Channel.value([ workflow base { channelValue | view{ "DEBUG1: $it" } - // : Channel[(String, Map[String, List[File]], params)] + // : Channel[(String, Map[String, List[File]], File)] // * it[0]: a string identifier // * it[1]: a map with a list of files // * it[2]: a file @@ -50,65 +47,125 @@ workflow base { // * it[1]: a map with a list of files | step3.run( + // test directives directives: [ publishDir: "output/foo" ], + // test debug + debug: true ) | view{ "DEBUG6: $it" } // : Channel[(String, File)] } -workflow map_variant { +workflow test_map_mapdata_mapid_arguments { channelValue | view{ "DEBUG1: $it" } | step1 | view{ "DEBUG2: $it" } | step2.run( - map: { [ it[0], [ "input1" : it[1], "input2" : it[2] ] ] }, + // test map + map: { [ it[0], [ "input1" : it[1], "input2" : it[2] ] ] } ) | view { "DEBUG3: $it" } | step3.run( - map: { [ it[0], [ "input": [ it[1].output1 , it[1].output2 ] ] ] }, - directives: [ - publishDir: "output/foo" - ], + // test id + mapId: {it + "_bar"}, + // test mapdata + mapData: { [ "input": [ it.output1 , it.output2 ] ] } ) - | view { "DEBUG4: $it" } + /* TESTING */ + | view{ "DEBUG4: $it"} + | toList() + | view { output_list -> + assert output_list.size() == 1 : "output channel should contain 1 event" + + def output = output_list[0] + assert output.size() == 2 : "outputs should contain two elements; [id, output]" + def id = output[0] + + // check id + assert id == "foo_bar" : "id should be foo_bar" + + // check final output file + def output_str = output[1].readLines().join("\n") + assert output_str.matches('^11 .*$') : 'output should match ^11 .*$' + + // return something to print + "DEBUG5: $output" + } } -workflow mapData_variant { - channelValue + +workflow test_fromstate_tostate_arguments { + Channel.fromList([ + [ + // id + "foo", + // data + [ + "input": file("${params.rootDir}/resources/lines*.txt"), + "lines3": file("${params.rootDir}/resources/lines3.txt") + ] + ] + ]) | view{ "DEBUG1: $it" } - | step1 - | view{ "DEBUG2: $it" } - | step2.run( - map: { [ it[0], [ "input1" : it[1], "input2" : it[2] ] ] }, - ) - | view { "DEBUG3: $it" } - | step3.run( - mapData: { [ "input": [ it.output1 , it.output2 ] ] }, - directives: [ - publishDir: "output/foo" - ], - ) - | view { "DEBUG4: $it" } -} -workflow debug_variant { - channelValue + // test fromstate and tostate with list[string] | step1.run( - debug: debug, + fromState: ["input"], + toState: ["output"], + auto: [simplifyOutput: false] ) + | view{ "DEBUG2: $it" } + + // test fromstate and tostate with map[string, string] | step2.run( - map: { [ it[0], [ "input1" : it[1], "input2" : it[2] ] ] }, - debug: debug, + fromState: ["input1": "output", "input2": "lines3"], + toState: ["step2_output1": "output1", "step2_output2": "output2"], + auto: [simplifyOutput: false] ) + | view{ "DEBUG3: $it" } + + // test fromstate and tostate with closure | step3.run( - mapData: { [ "input": [ it.output1 , it.output2 ] ] }, - directives: [ - publishDir: "output/foo" - ], - debug: debug, + fromState: { id, state -> + [ "input": [state.step2_output1, state.step2_output2] ] + }, + toState: { id, output, state -> + state + [ "step3_output": output.output ] + }, + auto: [simplifyOutput: false] ) - | view { "DEBUG4: $it" } -} + /* TESTING */ + | toList() + | view { output_list -> + assert output_list.size() == 1 : "output channel should contain 1 event" + + def output = output_list[0] + assert output.size() == 2 : "outputs should contain two elements; [id, state]" + def id = output[0] + def state = output[1] + + // check id + assert id == "foo" : "id should be foo" + + // check state + for (key in ["input", "lines3", "output", "step2_output1", "step2_output2", "step3_output"]) { + assert state.containsKey(key) : "state should contain key $key" + def value = state[key] + if (key == "input") { + assert value instanceof List : "state[input] should be a List" + } else { + assert value instanceof Path : "state[$key] should be a Path" + } + } + + // check final output file + def output_str = state["step3_output"].readLines().join("\n") + assert output_str.matches('^11 .*$') : 'output should match ^11 .*$' + + // return something to print + "DEBUG4: $output" + } +} \ No newline at end of file diff --git a/src/test/scala/io/viash/Tags.scala b/src/test/scala/io/viash/Tags.scala index 453a61adc..56e19c90d 100644 --- a/src/test/scala/io/viash/Tags.scala +++ b/src/test/scala/io/viash/Tags.scala @@ -6,4 +6,4 @@ object DockerTest extends Tag("io.viash.DockerTest") object NativeTest extends Tag("io.viash.NativeTest") -object NextFlowTest extends Tag("io.viash.NextFlowTest") +object NextflowTest extends Tag("io.viash.NextflowTest") diff --git a/src/test/scala/io/viash/TestingAllComponentsSuite.scala b/src/test/scala/io/viash/TestingAllComponentsSuite.scala index b60e08297..9433e4ced 100644 --- a/src/test/scala/io/viash/TestingAllComponentsSuite.scala +++ b/src/test/scala/io/viash/TestingAllComponentsSuite.scala @@ -2,8 +2,11 @@ package io.viash import io.viash.config.Config import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger +import org.scalatest.ParallelTestExecution -class TestingAllComponentsSuite extends AnyFunSuite { +class TestingAllComponentsSuite extends AnyFunSuite with ParallelTestExecution { + Logger.UseColorOverride.value = Some(false) def getTestResource(path: String) = getClass.getResource(path).toString val tests = List( @@ -16,6 +19,13 @@ class TestingAllComponentsSuite extends AnyFunSuite { ("executable", "config.vsh.yaml") ) + val multiples = List( + "boolean", + "integer", + "long", + "double", + ) + for ((name, file) <- tests) { val config = getTestResource(s"/test_languages/$name/$file") @@ -24,12 +34,43 @@ class TestingAllComponentsSuite extends AnyFunSuite { test(s"Testing $name platform native", NativeTest) { TestHelper.testMain("test", "-p", "native", config) } + + for (multiType <- multiples) { + test(s"Testing $name platform native, multiple $multiType", NativeTest) { + TestHelper.testMain( + "test", "-p", "native", config, + "-c", s""".functionality.argument_groups[.name == "Arguments"].arguments[.name == "--multiple" || .name == "multiple_pos"].type := "$multiType"""", + "-c", s""".functionality.test_resources[.type == "bash_script"].path := "../multi-$multiType.sh"""" + ) + } + } } test(s"Testing $name platform docker", DockerTest) { TestHelper.testMain("test", "-p", "docker", config) } + if (name != "executable") { + for (multiple <- multiples) { + test(s"Testing $name platform docker, multiple $multiple", DockerTest) { + TestHelper.testMain( + "test", "-p", "docker", config, + "-c", s""".functionality.argument_groups[.name == "Arguments"].arguments[.name == "--multiple" || .name == "multiple_pos"].type := "$multiple"""", + "-c", s""".functionality.test_resources[.type == "bash_script"].path := "../multi-$multiple.sh"""" + ) + } + } + + test(s"Testing $name platform docker, multiple file", DockerTest) { + TestHelper.testMain( + "test", "-p", "docker", config, + "-c", s""".functionality.argument_groups[.name == "Arguments"].arguments[.name == "--multiple" || .name == "multiple_pos"].type := "file"""", + "-c", s""".functionality.argument_groups[.name == "Arguments"].arguments[.name == "--multiple" || .name == "multiple_pos"].must_exist := false""", + "-c", s""".functionality.test_resources[.type == "bash_script"].path := "../multi-file.sh"""" + ) + } + } + test(s"Testing $name whether yaml parsing/unparsing is invertible") { import io.circe.syntax._ diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala index c27378334..a5a1f3152 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerChown.scala @@ -1,103 +1,50 @@ package io.viash.auxiliary import io.viash.{DockerTest, TestHelper} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.config.Config import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} import scala.io.Source +import io.viash.ConfigDeriver +import org.scalatest.ParallelTestExecution -class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll { +class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll with ParallelTestExecution { + Logger.UseColorOverride.value = Some(false) private val temporaryFolder = IO.makeTemp("viash_tester") private val tempFolStr = temporaryFolder.toString - - private val configDockerOptionsChownFile = getClass.getResource("/testbash/docker_options/config_chown.vsh.yaml").getPath - private val configDockerOptionsChownTwoOutputFile = getClass.getResource("/testbash/docker_options/config_chown_two_output.vsh.yaml").getPath - private val configDockerOptionsChownMultipleOutputFile = getClass.getResource("/testbash/docker_options/config_chown_multiple_output.vsh.yaml").getPath - - - def dockerChownGetOwner(dockerId: String): String = { - val localConfig = configDockerOptionsChownFile - val localFunctionality = Config.read(localConfig).functionality - val localExecutable = Paths.get(tempFolStr, localFunctionality.name).toFile - - // prepare the environment - TestHelper.testMain( - "build", - "-p", dockerId, - "-o", tempFolStr, - "--setup", "build", - localConfig - ) - - assert(localExecutable.exists) - assert(localExecutable.canExecute) - - // run the script - val output = Paths.get(tempFolStr, s"output_" + dockerId + ".txt").toFile - - Exec.run( - Seq( - localExecutable.toString, - localExecutable.toString, - "--real_number", "10.5", - "--whole_number=10", - "-s", "a string with a few spaces", - "--output", output.getPath - ) - ) - - assert(output.exists()) - - val owner = Files.getOwner(output.toPath) - owner.toString - } - - def dockerChownGetOwnerTwoOutputs(dockerId: String): (String,String) = { - val localConfig = configDockerOptionsChownTwoOutputFile - val localFunctionality = Config.read(localConfig).functionality - val localExecutable = Paths.get(tempFolStr, localFunctionality.name).toFile - - // prepare the environment - TestHelper.testMain( - "build", - "-p", dockerId, - "-o", tempFolStr, - "--setup", "build", - localConfig - ) - - assert(localExecutable.exists) - assert(localExecutable.canExecute) - - // run the script - val output = Paths.get(tempFolStr, "output_" + dockerId + ".txt").toFile - val output2 = Paths.get(tempFolStr, "output_" + dockerId +"_2.txt").toFile - - val _ = Exec.run( - Seq( - localExecutable.toString, - localExecutable.toString, - "--real_number", "10.5", - "--whole_number=10", - "-s", "a string with a few spaces", - "--output", output.getPath, - "--output2", output2.getPath - ) - ) - - assert(output.exists()) - assert(output2.exists()) - - val owner = Files.getOwner(output.toPath) - val owner2 = Files.getOwner(output2.toPath) - (owner.toString, owner2.toString) - } - - def dockerChownGetOwnerMultipleOutputs(dockerId: String): (String,String,String) = { - val localConfig = configDockerOptionsChownMultipleOutputFile + private val temporaryConfigFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + + private val configFile = getClass.getResource("/testbash/config.vsh.yaml").getPath + private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryConfigFolder) + + val singleOutputmods = List( + """.functionality.resources[.type == "bash_script"].path := "docker_options/code.sh"""", + ) + + val twoOutputsmods = List( + """.functionality.argument_groups[.name == "Arguments"].arguments += {name: "--output2", type: "file", direction: "output"}""", + """.functionality.resources[.type == "bash_script"].path := "docker_options/code_two_output.sh"""", + ) + + val multipleOutputsMods = List( + """del(.functionality.argument_groups[.name == "Arguments"].arguments[.name == "--multiple"])""", + """del(.functionality.argument_groups[.name == "Arguments"].arguments[.name == "multiple_pos"])""", + """.functionality.argument_groups[.name == "Arguments"].arguments[.name == "--output"].multiple := true""", + """.functionality.argument_groups[.name == "Arguments"].arguments += {name: "output_pos", type: "file", direction: "output", multiple: true}""", + """.functionality.resources[.type == "bash_script"].path := "docker_options/code_multiple_output.sh"""", + ) + + def dockerChownGetOwner(mods: List[String], amount: Int, dockerId: String, chown: Option[Boolean]): List[String] = { + assert(amount > 0) + assert(amount < 4) + + val extra = chown.map(b => s""", "chown": $b""" ).getOrElse("") + val platformMod = s""".platforms := [{"type": "docker", "image": "bash:3.2", "id": "$dockerId"$extra}]""" + val modsWithPlatform = mods :+ platformMod + val localConfig = configDeriver.derive(modsWithPlatform, dockerId) val localFunctionality = Config.read(localConfig).functionality val localExecutable = Paths.get(tempFolStr, localFunctionality.name).toFile @@ -118,6 +65,12 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll { val output2 = Paths.get(tempFolStr, "output_" + dockerId +"_2.txt").toFile val output3 = Paths.get(tempFolStr, "output_" + dockerId +"_3.txt").toFile + val outputParams = amount match { + case 1 => Seq("--output", output.getPath) + case 2 => Seq("--output", output.getPath, "--output2", output2.getPath) + case 3 => Seq("--output", output.getPath, output2.getPath, output3.getPath) + } + Exec.run( Seq( localExecutable.toString, @@ -125,87 +78,94 @@ class MainBuildAuxiliaryDockerChown extends AnyFunSuite with BeforeAndAfterAll { "--real_number", "10.5", "--whole_number=10", "-s", "a string with a few spaces", - "--output", output.getPath, output2.getPath, output3.getPath - ) + ) ++ outputParams ) - assert(output.exists()) - assert(output2.exists()) - assert(output3.exists()) - - val owner = Files.getOwner(output.toPath) - val owner2 = Files.getOwner(output2.toPath) - val owner3 = Files.getOwner(output3.toPath) - (owner.toString, owner2.toString, owner3.toString) + val outputList = List(output, output2, output3).take(amount) + outputList.foreach(output => assert(output.exists())) + outputList.map(file => Files.getOwner(file.toPath).toString) } test("Test default behaviour when chown is not specified", DockerTest) { - val owner = dockerChownGetOwner("chown_default") - assert(owner.nonEmpty) - assert(owner != "root") + val owners = dockerChownGetOwner(singleOutputmods, 1, "chown_default", None) + assert(owners.length == 1) + owners.foreach(owner => { + assert(owner.nonEmpty) + assert(owner != "root") + }) } test("Test default behaviour when chown is set to true", DockerTest) { - val owner = dockerChownGetOwner("chown_true") - assert(owner.nonEmpty) - assert(owner != "root") + val owners = dockerChownGetOwner(singleOutputmods, 1, "chown_true", Some(true)) + assert(owners.length == 1) + owners.foreach(owner => { + assert(owner.nonEmpty) + assert(owner != "root") + }) } test("Test default behaviour when chown is set to false", DockerTest) { - val owner = dockerChownGetOwner("chown_false") - assert(owner == "root") + val owners = dockerChownGetOwner(singleOutputmods, 1, "chown_false", Some(false)) + assert(owners.length == 1) + owners.foreach(owner => { + assert(owner == "root") + }) } test("Test default behaviour when chown is not specified with two output files", DockerTest) { - val owner = dockerChownGetOwnerTwoOutputs("two_chown_default") - assert(owner._1.nonEmpty) - assert(owner._2.nonEmpty) - assert(owner._1 != "root") - assert(owner._2 != "root") + val owners = dockerChownGetOwner(twoOutputsmods, 2, "two_chown_default", None) + assert(owners.length == 2) + owners.foreach(owner => { + assert(owner.nonEmpty) + assert(owner != "root") + }) } test("Test default behaviour when chown is set to true with two output files", DockerTest) { - val owner = dockerChownGetOwnerTwoOutputs("two_chown_true") - assert(owner._1.nonEmpty) - assert(owner._2.nonEmpty) - assert(owner._1 != "root") - assert(owner._2 != "root") + val owners = dockerChownGetOwner(twoOutputsmods, 2, "two_chown_true", Some(true)) + assert(owners.length == 2) + owners.foreach(owner => { + assert(owner.nonEmpty) + assert(owner != "root") + }) } test("Test default behaviour when chown is set to false with two output files", DockerTest) { - val owner = dockerChownGetOwnerTwoOutputs("two_chown_false") - assert(owner._1 == "root") - assert(owner._2 == "root") + val owners = dockerChownGetOwner(twoOutputsmods, 2, "two_chown_false", Some(false)) + assert(owners.length == 2) + owners.foreach(owner => { + assert(owner == "root") + }) } test("Test default behaviour when chown is not specified with multiple output files", DockerTest) { - val owner = dockerChownGetOwnerMultipleOutputs("multiple_chown_default") - assert(owner._1.nonEmpty) - assert(owner._2.nonEmpty) - assert(owner._3.nonEmpty) - assert(owner._1 != "root") - assert(owner._2 != "root") - assert(owner._3 != "root") + val owners = dockerChownGetOwner(multipleOutputsMods, 3, "multiple_chown_default", None) + assert(owners.length == 3) + owners.foreach(owner => { + assert(owner.nonEmpty) + assert(owner != "root") + }) } test("Test default behaviour when chown is set to true with multiple output files", DockerTest) { - val owner = dockerChownGetOwnerMultipleOutputs("multiple_chown_true") - assert(owner._1.nonEmpty) - assert(owner._2.nonEmpty) - assert(owner._3.nonEmpty) - assert(owner._1 != "root") - assert(owner._2 != "root") - assert(owner._3 != "root") + val owners = dockerChownGetOwner(multipleOutputsMods, 3, "multiple_chown_true", Some(true)) + assert(owners.length == 3) + owners.foreach(owner => { + assert(owner.nonEmpty) + assert(owner != "root") + }) } test("Test default behaviour when chown is set to false with multiple output files", DockerTest) { - val owner = dockerChownGetOwnerMultipleOutputs("multiple_chown_false") - assert(owner._1 == "root") - assert(owner._2 == "root") - assert(owner._3 == "root") + val owners = dockerChownGetOwner(multipleOutputsMods, 3, "multiple_chown_false", Some(false)) + assert(owners.length == 3) + owners.foreach(owner => { + assert(owner == "root") + }) } override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) + IO.deleteRecursively(temporaryConfigFolder) } -} \ No newline at end of file +} diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala index cb519d5f9..9cc063001 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerRequirements.scala @@ -1,31 +1,94 @@ package io.viash.auxiliary import io.viash.config.Config -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.{DockerTest, TestHelper} import org.scalatest.BeforeAndAfterAll -import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.funsuite.FixtureAnyFunSuite import java.nio.file.{Files, Paths} import scala.io.Source +import io.viash.ConfigDeriver -class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAfterAll { +abstract class AbstractMainBuildAuxiliaryDockerRequirements extends FixtureAnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) private val temporaryFolder = IO.makeTemp("viash_tester") - private val tempFolStr = temporaryFolder.toString + protected val tempFolStr = temporaryFolder.toString + private val temporaryConfigFolder = IO.makeTemp("viash_tester_configs") private val configRequirementsFile = getClass.getResource(s"/testbash/auxiliary_requirements/config_requirements.vsh.yaml").getPath private val functionalityRequirements = Config.read(configRequirementsFile).functionality - private val executableRequirementsFile = Paths.get(tempFolStr, functionalityRequirements.name).toFile + protected val executableRequirementsFile = Paths.get(tempFolStr, functionalityRequirements.name).toFile + + protected val configDeriver = ConfigDeriver(Paths.get(configRequirementsFile), temporaryConfigFolder) + + protected val image = "bash:3.2" + protected val dockerTag = "viash_requirements_testbench" + + case class FixtureParam() + + // Fixture will remove the docker image before starting and remove it again after finishing + def withFixture(test: OneArgTest) = { + // remove docker if it exists + removeDockerImage(dockerTag) + assert(!checkDockerImageExists(dockerTag)) + + val theFixture = FixtureParam() + + val outcome = withFixture(test.toNoArgTest(theFixture)) // "loan" the fixture to the test + + // Tests finished, remove docker image + removeDockerImage(dockerTag) + + outcome + } + + def checkDockerImageExists(name: String): Boolean = { + val out = Exec.runCatch( + Seq("docker", "images", name) + ) + val regex = s"$name\\s*latest".r + regex.findFirstIn(out.output).isDefined + } + + def removeDockerImage(name: String): Unit = { + Exec.runCatch( + Seq("docker", "rmi", name, "-f") + ) + } + + def derivePlatformConfig(setup: Option[String], test_setup: Option[String], name: String): String = { + val setupStr = setup.map(s => s""", "setup": $s""").getOrElse("") + val testSetupStr = test_setup.map(s => s""", "test_setup": $s""").getOrElse("") + + configDeriver.derive( + s""".platforms := [{ "type": "docker", "image": "$image", "target_image": "$dockerTag" $setupStr $testSetupStr }]""", + name + ) + } + + override def afterAll(): Unit = { + IO.deleteRecursively(temporaryFolder) + IO.deleteRecursively(temporaryConfigFolder) + } +} + +class MainBuildAuxiliaryDockerRequirementsApk extends AbstractMainBuildAuxiliaryDockerRequirements { + override val dockerTag = "viash_requirements_testbench_apk" + override val image = "bash:3.2" + + test("setup; check base image for apk still does not contain the fortune package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "apk_base") - test("setup; check base image for apk still does not contain the fortune package", DockerTest) { TestHelper.testMain( "build", - "-p", "viash_requirement_apk_base", + "-p", "docker", "-o", tempFolStr, "--setup", "build", - configRequirementsFile + newConfigFilePath ) + assert(checkDockerImageExists(dockerTag)) assert(executableRequirementsFile.exists) assert(executableRequirementsFile.canExecute) @@ -39,25 +102,17 @@ class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAft assert(output.output == "") } - test("setup; check docker requirements using apk to add the fortune package", DockerTest) { - val tag = "viash_requirement_apk" - - // remove docker if it exists - removeDockerImage(tag) - assert(!checkDockerImageExists(tag)) + test("setup; check docker requirements using apk to add the fortune package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "apk", "packages": ["fortune"] }]"""), None, "apk_fortune") - // build viash wrapper with --setup TestHelper.testMain( "build", - "-p", "viash_requirement_apk", "-o", tempFolStr, "--setup", "build", - configRequirementsFile + newConfigFilePath ) - // verify docker exists - assert(checkDockerImageExists(tag)) - + assert(checkDockerImageExists(dockerTag)) assert(executableRequirementsFile.exists) assert(executableRequirementsFile.canExecute) @@ -69,52 +124,72 @@ class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAft ) assert(output.output == "/usr/bin/fortune\n") - - // Tests finished, remove docker image - removeDockerImage(tag) } - test("setup; check base image for apt still does not contain the cowsay package", DockerTest) { + test("setup; check docker requirements using apk but with an empty list", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "apk", "packages": [] }]"""), None, "apk_empty") + TestHelper.testMain( "build", - "-p", "viash_requirement_apt_base", "-o", tempFolStr, "--setup", "build", - configRequirementsFile + newConfigFilePath ) + assert(checkDockerImageExists(dockerTag)) assert(executableRequirementsFile.exists) assert(executableRequirementsFile.canExecute) val output = Exec.runCatch( Seq( executableRequirementsFile.toString, - "--which", "cowsay" + "--which", "fortune" ) ) assert(output.output == "") } +} - test("setup; check docker requirements using apt to add the cowsay package", DockerTest) { - val tag = "viash_requirement_apt" +class MainBuildAuxiliaryDockerRequirementsApt extends AbstractMainBuildAuxiliaryDockerRequirements { + override val dockerTag = "viash_requirements_testbench_apt" + override val image = "debian:bullseye-slim" - // remove docker if it exists - removeDockerImage(tag) - assert(!checkDockerImageExists(tag)) + test("setup; check base image for apt still does not contain the cowsay package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "apt_base") - // build viash wrapper with --setup - val _ = TestHelper.testMain( + TestHelper.testMain( "build", - "-p", "viash_requirement_apt", "-o", tempFolStr, "--setup", "build", - configRequirementsFile + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/games/cowsay" + ) ) - // verify docker exists - assert(checkDockerImageExists(tag)) + assert(output.output.contains("/usr/games/cowsay doesn't exist.")) + } + + test("setup; check docker requirements using apt to add the cowsay package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "apt", "packages": ["cowsay"] }]"""), None, "apt_cowsay") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + assert(checkDockerImageExists(dockerTag)) assert(executableRequirementsFile.exists) assert(executableRequirementsFile.canExecute) @@ -125,20 +200,355 @@ class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAft ) ) - assert(output.output == "/usr/games/cowsay exists.\n") + assert(output.output.contains("/usr/games/cowsay exists.")) + } - // Tests finished, remove docker image - removeDockerImage(tag) + test("setup; check docker requirements using apt but with an empty list", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "apt", "packages": [] }]"""), None, "apt_empty") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/games/cowsay" + ) + ) + + assert(output.output.contains("/usr/games/cowsay doesn't exist.")) + } +} + +class MainBuildAuxiliaryDockerRequirementsYum extends AbstractMainBuildAuxiliaryDockerRequirements{ + override val dockerTag = "viash_requirements_testbench_yum" + override val image = "fedora:38" + + test("setup; check base image for yum still does not contain the which package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "yum_base") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/bin/which" + ) + ) + + assert(output.output.contains("/usr/bin/which doesn't exist.")) + } + + test("setup; check docker requirements using yum to add the which package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "yum", "packages": ["which"] }]"""), None, "yum_which") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/bin/which" + ) + ) + + assert(output.output.contains("/usr/bin/which exists.")) + } + + test("setup; check docker requirements using yum but with an empty list", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "yum", "packages": [] }]"""), None, "yum_empty") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/bin/which" + ) + ) + + assert(output.output.contains("/usr/bin/which doesn't exist.")) + } +} + +class MainBuildAuxiliaryDockerRequirementsRuby extends AbstractMainBuildAuxiliaryDockerRequirements{ + override val dockerTag = "viash_requirements_testbench_ruby" + override val image = "ruby:slim-bullseye" + + test("setup; check base image for yum still does not contain the which package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "ruby_base") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/bundle/gems/tzinfo-2.0.4/lib/tzinfo.rb" + ) + ) + + assert(output.output.contains("/usr/local/bundle/gems/tzinfo-2.0.4/lib/tzinfo.rb doesn't exist.")) + } + + test("setup; check docker requirements using yum to add the tzinfo package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "ruby", "packages": ["tzinfo:2.0.4"] }]"""), None, "ruby_tzinfo") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/bundle/gems/tzinfo-2.0.4/lib/tzinfo.rb" + ) + ) + + assert(output.output.contains("/usr/local/bundle/gems/tzinfo-2.0.4/lib/tzinfo.rb exists.")) + } + + test("setup; check docker requirements using yum but with an empty list", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "ruby", "packages": [] }]"""), None, "ruby_empty") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/bundle/gems/tzinfo-2.0.4/lib/tzinfo.rb" + ) + ) + + assert(output.output.contains("/usr/local/bundle/gems/tzinfo-2.0.4/lib/tzinfo.rb doesn't exist.")) + } +} + +class MainBuildAuxiliaryDockerRequirementsR extends AbstractMainBuildAuxiliaryDockerRequirements{ + override val dockerTag = "viash_requirements_testbench_r" + override val image = "r-base:4.3.1" + + test("setup; check base image for r still does not contain the glue package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "r_base") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/lib/R/site-library/glue/R/glue" + ) + ) + + assert(output.output.contains("/usr/local/lib/R/site-library/glue/R/glue doesn't exist.")) + } + + test("setup; check docker requirements using r to add the glue package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "r", "packages": ["glue"] }]"""), None, "r_glue") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/lib/R/site-library/glue/R/glue" + ) + ) + + assert(output.output.contains("/usr/local/lib/R/site-library/glue/R/glue exists.")) + } + + test("setup; check docker requirements using r but with an empty list", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "r", "packages": [] }]"""), None, "r_empty") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/lib/R/site-library/glue/R/glue" + ) + ) + + assert(output.output.contains("/usr/local/lib/R/site-library/glue/R/glue doesn't exist.")) } +} +class MainBuildAuxiliaryDockerRequirementsRBioc extends AbstractMainBuildAuxiliaryDockerRequirements{ + override val dockerTag = "viash_requirements_testbench_rbioc" + override val image = "r-base:4.3.1" + + test("setup; check base image for r-bioc still does not contain the BiocGenerics package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "rbioc_base") - test("test_setup; check the fortune package isn't added for the build option", DockerTest) { TestHelper.testMain( "build", - "-p", "viash_requirement_apk_test_setup", "-o", tempFolStr, "--setup", "build", - configRequirementsFile + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/lib/R/site-library/BiocGenerics/R/BiocGenerics" + ) + ) + + assert(output.output.contains("/usr/local/lib/R/site-library/BiocGenerics/R/BiocGenerics doesn't exist.")) + } + + test("setup; check docker requirements using r to add the BiocGenerics package", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "r", "bioc": ["BiocGenerics"] }]"""), None, "rbioc_biocgenerics") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/lib/R/site-library/BiocGenerics/R/BiocGenerics" + ) + ) + + assert(output.output.contains("/usr/local/lib/R/site-library/BiocGenerics/R/BiocGenerics exists.")) + } + + test("setup; check docker requirements using r but with an empty list", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(Some("""[{ "type": "r", "bioc": [] }]"""), None, "rbioc_empty") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath + ) + + assert(checkDockerImageExists(dockerTag)) + assert(executableRequirementsFile.exists) + assert(executableRequirementsFile.canExecute) + + val output = Exec.runCatch( + Seq( + executableRequirementsFile.toString, + "--file", "/usr/local/lib/R/site-library/BiocGenerics/R/BiocGenerics" + ) + ) + + assert(output.output.contains("/usr/local/lib/R/site-library/BiocGenerics/R/BiocGenerics doesn't exist.")) + } +} + + +class MainBuildAuxiliaryDockerRequirementsApkTest extends AbstractMainBuildAuxiliaryDockerRequirements { + override val dockerTag = "viash_requirements_testbench_apktest" + override val image = "bash:3.2" + + test("test_setup; check the fortune package isn't added for the build option", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, Some("""[{ "type": "apk", "packages": ["fortune"] }]"""), "apk_test_fortune_build") + + TestHelper.testMain( + "build", + "-o", tempFolStr, + "--setup", "build", + newConfigFilePath ) assert(executableRequirementsFile.exists) @@ -154,11 +564,12 @@ class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAft assert(output.output == "") } - test("test_setup; check the fortune package is added for the test option", DockerTest) { + test("test_setup; check the fortune package is added for the test option", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, Some("""[{ "type": "apk", "packages": ["fortune"] }]"""), "apk_test_fortune_test") + val testText = TestHelper.testMain( "test", - "-p", "viash_requirement_apk_test_setup", - configRequirementsFile + newConfigFilePath ) assert(testText.contains("Running tests in temporary directory: ")) @@ -166,13 +577,13 @@ class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAft assert(testText.contains("Cleaning up temporary directory")) } - test("test_setup; check the fortune package is not added for the test option when not specified", DockerTest) { + test("test_setup; check the fortune package is not added for the test option when not specified", DockerTest) { f => + val newConfigFilePath = derivePlatformConfig(None, None, "apk_base_test") val testOutput = TestHelper.testMainException2[RuntimeException]( "test", - "-p", "viash_requirement_apk_base", "-k", "false", - configRequirementsFile + newConfigFilePath ) assert(testOutput.exceptionText == "Only 0 out of 1 test scripts succeeded!") @@ -181,25 +592,4 @@ class MainBuildAuxiliaryDockerRequirements extends AnyFunSuite with BeforeAndAft assert(testOutput.output.contains("ERROR! Only 0 out of 1 test scripts succeeded!")) assert(testOutput.output.contains("Cleaning up temporary directory")) } - - def checkDockerImageExists(name: String): Boolean = { - val out = Exec.runCatch( - Seq("docker", "images", name) - ) - - // print(out) - val regex = s"$name\\s*latest".r - - regex.findFirstIn(out.output).isDefined - } - - def removeDockerImage(name: String): Unit = { - Exec.runCatch( - Seq("docker", "rmi", name, "-f") - ) - } - - override def afterAll(): Unit = { - IO.deleteRecursively(temporaryFolder) - } -} \ No newline at end of file +} diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala index 9afb9ae2b..748ee1921 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerResourceCopying.scala @@ -2,13 +2,15 @@ package io.viash.auxiliary import io.viash.{DockerTest, TestHelper} import io.viash.config.Config -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} +import io.viash.ConfigDeriver class MainBuildAuxiliaryDockerResourceCopying extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) private val temporaryFolder = IO.makeTemp("viash_tester") private val tempFolStr = temporaryFolder.toString @@ -17,9 +19,8 @@ class MainBuildAuxiliaryDockerResourceCopying extends AnyFunSuite with BeforeAnd private val functionality = Config.read(configFile).functionality private val executable = Paths.get(tempFolStr, functionality.name).toFile - private val configResourcesUnsupportedProtocolFile = getClass.getResource("/testbash/auxiliary_resource/config_resource_unsupported_protocol.vsh.yaml").getPath - - + private val temporaryConfigFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryConfigFolder) test("Check resources are copied from and to the correct location") { @@ -74,6 +75,7 @@ class MainBuildAuxiliaryDockerResourceCopying extends AnyFunSuite with BeforeAnd } test("Check resources with unsupported format") { + val configResourcesUnsupportedProtocolFile = configDeriver.derive(""".functionality.resources := [{type: "bash_script", path: "./check_bash_version.sh"}, {path: "ftp://ftp.ubuntu.com/releases/robots.txt"}]""", "config_resource_unsupported_protocol").toString // generate viash script val testOutput = TestHelper.testMainException2[RuntimeException]( "build", @@ -87,5 +89,6 @@ class MainBuildAuxiliaryDockerResourceCopying extends AnyFunSuite with BeforeAnd override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) + IO.deleteRecursively(temporaryConfigFolder) } } \ No newline at end of file diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala index 4f558cff3..bc51630dd 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryDockerTag.scala @@ -2,7 +2,7 @@ package io.viash.auxiliary import io.viash.{DockerTest, TestHelper} import io.viash.config.Config -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -10,6 +10,7 @@ import java.nio.file.{Files, Paths} import scala.io.Source class MainBuildAuxiliaryDockerTag extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) private val temporaryFolder = IO.makeTemp("viash_tester") private val tempFolStr = temporaryFolder.toString diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala index 5e81fee68..70939aecb 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeParameterCheck.scala @@ -7,12 +7,13 @@ import java.nio.file.{Paths, Files, StandardCopyOption} import io.viash.config.Config import scala.io.Source -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.TestHelper import java.nio.file.Path import scala.annotation.meta.param class MainBuildAuxiliaryNativeParameterCheck extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource("/testbash/auxiliary_requirements/parameter_check.vsh.yaml").getPath private val loopConfigFile = getClass.getResource("/testbash/auxiliary_requirements/parameter_check_loop.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala index 9ea9914e6..3ee71f17f 100644 --- a/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala +++ b/src/test/scala/io/viash/auxiliary/MainBuildAuxiliaryNativeUnknownParameter.scala @@ -7,10 +7,11 @@ import java.nio.file.Paths import io.viash.config.Config import scala.io.Source -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.TestHelper class MainBuildAuxiliaryNativeUnknownParameter extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource("/testbash/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala index 625215aa0..3064e6955 100644 --- a/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala +++ b/src/test/scala/io/viash/auxiliary/MainRunVersionSwitch.scala @@ -1,45 +1,30 @@ package io.viash.auxiliary import io.viash.{NativeTest, TestHelper, Main} -import io.viash.helpers.IO +import io.viash.helpers.{IO, SysEnv, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} import scala.reflect.io.Directory import java.io.ByteArrayOutputStream -import java.security.Permission - -// Use SecurityManager to capture System.exit codes set by Scallop as this would cancel our testbench -sealed case class ExitException(status: Int) extends SecurityException("System.exit() is not allowed") { -} - -sealed class NoExitSecurityManager extends SecurityManager { - override def checkPermission(perm: Permission): Unit = {} - - override def checkPermission(perm: Permission, context: Object): Unit = {} - - override def checkExit(status: Int): Unit = { - super.checkExit(status) - throw ExitException(status) - } -} +import io.viash.exceptions.ExitException class MainRunVersionSwitch extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) - override def beforeAll(): Unit = System.setSecurityManager(new NoExitSecurityManager()) - - override def afterAll(): Unit = System.setSecurityManager(null) + test("Verify VIASH_VERSION is unset") { + assert(SysEnv.viashVersion.isEmpty) + } - def setEnv(key: String, value: String) = { - val field = System.getenv().getClass.getDeclaredField("m") - field.setAccessible(true) - val map = field.get(System.getenv()).asInstanceOf[java.util.Map[java.lang.String, java.lang.String]] - map.put(key, value) + test("Can override variables") { + SysEnv.set("VIASH_VERSION", "foo") + assert(SysEnv.viashVersion == Some("foo")) } - test("Verify VIASH_VERSION is undefined by default", NativeTest) { - assert(sys.env.get("VIASH_VERSION").isDefined == false) + test("Can erase variables") { + SysEnv.remove("VIASH_VERSION") + assert(SysEnv.viashVersion.isEmpty) } test("Check version without specifying the version to run", NativeTest) { @@ -64,9 +49,9 @@ class MainRunVersionSwitch extends AnyFunSuite with BeforeAndAfterAll { test("Check version with specifying the version to run", NativeTest) { - setEnv("VIASH_VERSION", "0.6.6") + SysEnv.set("VIASH_VERSION", "0.6.6") - val version = sys.env.get("VIASH_VERSION") + val version = SysEnv.viashVersion assert(version == Some("0.6.6")) val arguments = Seq("--version") @@ -87,9 +72,9 @@ class MainRunVersionSwitch extends AnyFunSuite with BeforeAndAfterAll { test("Check version with specifying '-' as the version to run", NativeTest) { - setEnv("VIASH_VERSION", "-") + SysEnv.set("VIASH_VERSION", "-") - val version = sys.env.get("VIASH_VERSION") + val version = SysEnv.viashVersion assert(version == Some("-")) val arguments = Seq("--version") @@ -113,12 +98,12 @@ class MainRunVersionSwitch extends AnyFunSuite with BeforeAndAfterAll { test("Check version with specifying an invalid version", NativeTest) { // remove the 'invalid' viash version if it already exists - val path = Main.viashHome.resolve("releases").resolve("invalid").resolve("viash") + val path = Paths.get(SysEnv.viashHome).resolve("releases").resolve("invalid").resolve("viash") Files.deleteIfExists(path) - setEnv("VIASH_VERSION", "invalid") + SysEnv.set("VIASH_VERSION", "invalid") - val version = sys.env.get("VIASH_VERSION") + val version = SysEnv.viashVersion assert(version == Some("invalid")) val arguments = Seq("--version") @@ -140,4 +125,8 @@ class MainRunVersionSwitch extends AnyFunSuite with BeforeAndAfterAll { assert(caught.getMessage().contains("Could not download file: https://github.com/viash-io/viash/releases/download/invalid/viash")) } + override def afterAll(): Unit = { + SysEnv.remove("VIASH_VERSION") + } + } diff --git a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala index c91b3ef8a..fc1d4cdf4 100644 --- a/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala +++ b/src/test/scala/io/viash/auxiliary/MainTestAuxiliaryDockerResourceCopy.scala @@ -1,16 +1,19 @@ package io.viash.auxiliary import io.viash.{DockerTest, TestHelper} -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} import scala.reflect.io.Directory +import io.viash.ConfigDeriver class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfterAll { - private val configResourcesCopyFile = getClass.getResource("/testbash/auxiliary_resource/config_resource_test.vsh.yaml").getPath - private val configResourcesUnsupportedProtocolFile = getClass.getResource("/testbash/auxiliary_resource/config_resource_unsupported_protocol.vsh.yaml").getPath + Logger.UseColorOverride.value = Some(false) + private val configFile = getClass.getResource("/testbash/auxiliary_resource/config_resource_test.vsh.yaml").getPath + private val temporaryConfigFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryConfigFolder) test("Check resources are copied from and to the correct location", DockerTest) { @@ -30,7 +33,7 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte "test", "-p", "docker", "-k", "true", - configResourcesCopyFile + configFile ) // basic checks to see if standard test/build was correct @@ -78,6 +81,7 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte } test("Check resources with unsupported format", DockerTest) { + val configResourcesUnsupportedProtocolFile = configDeriver.derive(""".functionality.resources := [{type: "bash_script", path: "./check_bash_version.sh"}, {path: "ftp://ftp.ubuntu.com/releases/robots.txt"}]""", "config_resource_unsupported_protocol").toString // generate viash script val testOutput = TestHelper.testMainException2[RuntimeException]( "test", @@ -93,7 +97,7 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte assert(!testOutput.output.contains("WARNING! No tests found!")) assert(!testOutput.output.contains("Cleaning up temporary directory")) - checkTempDirAndRemove(testOutput.output, true) + checkTempDirAndRemove(testOutput.output, true, "viash_test_auxiliary_resources") } /** @@ -128,4 +132,8 @@ class MainTestAuxiliaryDockerResourceCopy extends AnyFunSuite with BeforeAndAfte // folder should always have been removed at this stage assert(!tempFolder.exists) } + + override def afterAll(): Unit = { + IO.deleteRecursively(temporaryConfigFolder) + } } diff --git a/src/test/scala/io/viash/config_mods/AppendTest.scala b/src/test/scala/io/viash/config_mods/AppendTest.scala index 97823164e..beefe8225 100644 --- a/src/test/scala/io/viash/config_mods/AppendTest.scala +++ b/src/test/scala/io/viash/config_mods/AppendTest.scala @@ -5,8 +5,10 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ import io.circe.yaml.parser.parse +import io.viash.helpers.Logger class AppendTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsing test("parse append command") { val expected = ConfigMods(List( @@ -72,4 +74,26 @@ class AppendTest extends AnyFunSuite { val res2 = cmd2.apply(baseJson, false) assert(res2 == expected2) } + + test("test append array") { + val expected2: Json = parse( + """foo: bar + |baz: 123 + |list_of_stuff: [4, 5, 6, 1, 2, 3] + |""".stripMargin).toOption.get + val cmd2 = ConfigModParser.block.parse(""".list_of_stuff += [ 1, 2, 3 ]""") + val res2 = cmd2.apply(baseJson, false) + assert(res2 == expected2) + } + + test("test prepend array") { + val expected2: Json = parse( + """foo: bar + |baz: 123 + |list_of_stuff: [1, 2, 3, 4, 5, 6] + |""".stripMargin).toOption.get + val cmd2 = ConfigModParser.block.parse(""".list_of_stuff +0= [ 1, 2, 3 ]""") + val res2 = cmd2.apply(baseJson, false) + assert(res2 == expected2) + } } \ No newline at end of file diff --git a/src/test/scala/io/viash/config_mods/AssignTest.scala b/src/test/scala/io/viash/config_mods/AssignTest.scala index 53576184a..c61f8c5f5 100644 --- a/src/test/scala/io/viash/config_mods/AssignTest.scala +++ b/src/test/scala/io/viash/config_mods/AssignTest.scala @@ -4,7 +4,11 @@ import io.circe.Json import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ +import io.circe.yaml.parser.parse +import io.viash.helpers.Logger + class AssignTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsing test("parse assign command with only attributes") { val expected = ConfigMods(List( @@ -111,5 +115,222 @@ class AssignTest extends AnyFunSuite { } // testing functionality - // TODO + val baseJson: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 2 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + + test("test assign single entry from a list #1") { + val expected1: Json = parse( + """foo: + | - name: quux + | a: 5 + | - name: baz + | a: 2 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a == 1] := { name: "quux", a: 5 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign single entry from a list #2") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: quux + | a: 6 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a == 2] := { name: "quux", a: 6 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign single entry from a list #3") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 2 + | - name: quux + | a: 7 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a == 3] := { name: "quux", a: 7 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign field from a single list entry #1") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 5 + | - name: baz + | a: 2 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a == 1].a := 5""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign field from a single list entry #2") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 6 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a == 2].a := 6""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign field from a single list entry #3") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 2 + | - name: qux + | a: 7 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a == 3].a := 7""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign multiple entries from a list #1") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: quux + | a: 5 + | - name: quux + | a: 5 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a != 1] := { name: "quux", a: 5 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign multiple entries from a list #2") { + val expected1: Json = parse( + """foo: + | - name: quux + | a: 6 + | - name: baz + | a: 2 + | - name: quux + | a: 6 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a != 2] := { name: "quux", a: 6 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign multiple entries from a list #3") { + val expected1: Json = parse( + """foo: + | - name: quux + | a: 7 + | - name: quux + | a: 7 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a != 3] := { name: "quux", a: 7 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign field from multiple list entries #1") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 5 + | - name: qux + | a: 5 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a != 1].a := 5""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign field from multiple list entries #2") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 6 + | - name: baz + | a: 2 + | - name: qux + | a: 6 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a != 2].a := 6""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + test("test assign field from multiple list entries #3") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 7 + | - name: baz + | a: 7 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[.a != 3].a := 7""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign all entries from a list") { + val expected1: Json = parse( + """foo: + | - name: quux + | a: 5 + | - name: quux + | a: 5 + | - name: quux + | a: 5 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[true] := { name: "quux", a: 5 }""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test assign field from all list entries") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 5 + | - name: baz + | a: 5 + | - name: qux + | a: 5 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse(""".foo[true].a := 5""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } } \ No newline at end of file diff --git a/src/test/scala/io/viash/config_mods/ConditionTest.scala b/src/test/scala/io/viash/config_mods/ConditionTest.scala index f6ca8acbe..23120fdd6 100644 --- a/src/test/scala/io/viash/config_mods/ConditionTest.scala +++ b/src/test/scala/io/viash/config_mods/ConditionTest.scala @@ -5,8 +5,10 @@ import io.circe.syntax._ import io.circe.yaml.parser.parse import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger class ConditionSuite extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsers // TODO diff --git a/src/test/scala/io/viash/config_mods/ConfigModsTest.scala b/src/test/scala/io/viash/config_mods/ConfigModsTest.scala index 6e3a1a4d2..97d424646 100644 --- a/src/test/scala/io/viash/config_mods/ConfigModsTest.scala +++ b/src/test/scala/io/viash/config_mods/ConfigModsTest.scala @@ -3,8 +3,10 @@ package io.viash.config_mods import io.circe.Json import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ +import io.viash.helpers.Logger class ConfigModsTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsers test("parsing multiple commands in one go") { val expected = ConfigMods( diff --git a/src/test/scala/io/viash/config_mods/DeleteTest.scala b/src/test/scala/io/viash/config_mods/DeleteTest.scala index 5b892e7f6..a15032e56 100644 --- a/src/test/scala/io/viash/config_mods/DeleteTest.scala +++ b/src/test/scala/io/viash/config_mods/DeleteTest.scala @@ -3,8 +3,12 @@ package io.viash.config_mods import io.circe.Json import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ +import io.viash.helpers.Logger + +import io.circe.yaml.parser.parse class DeleteTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsing test("parsing delete command") { val expected = ConfigMods(List( @@ -18,5 +22,187 @@ class DeleteTest extends AnyFunSuite { } // testing functionality - // TODO + val baseJson: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 2 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + + test("test delete single entry from a list #1") { + val expected1: Json = parse( + """foo: + | - name: baz + | a: 2 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a == 1])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete single entry from a list #2") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a == 2])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete single entry from a list #3") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 2 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a == 3])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete field from a single list entry #1") { + val expected1: Json = parse( + """foo: + | - name: bar + | - name: baz + | a: 2 + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a == 1].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete field from a single list entry #2") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a == 2].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete field from a single list entry #3") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | a: 2 + | - name: qux + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a == 3].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete multiple entries from a list #1") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a != 1])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete multiple entries from a list #2") { + val expected1: Json = parse( + """foo: + | - name: baz + | a: 2 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a != 2])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete multiple entries from a list #3") { + val expected1: Json = parse( + """foo: + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a != 3])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete field from multiple list entries #1") { + val expected1: Json = parse( + """foo: + | - name: bar + | a: 1 + | - name: baz + | - name: qux + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a != 1].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete field from multiple list entries #2") { + val expected1: Json = parse( + """foo: + | - name: bar + | - name: baz + | a: 2 + | - name: qux + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a != 2].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + test("test delete field from multiple list entries #3") { + val expected1: Json = parse( + """foo: + | - name: bar + | - name: baz + | - name: qux + | a: 3 + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[.a != 3].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete all entries from a list") { + val expected1: Json = parse( + """foo: [] + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[true])""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + + test("test delete field from all list entries") { + val expected1: Json = parse( + """foo: + | - name: bar + | - name: baz + | - name: qux + |""".stripMargin).toOption.get + val cmd1 = ConfigModParser.block.parse("""del(.foo[true].a)""") + val res1 = cmd1.apply(baseJson, false) + assert(res1 == expected1) + } + } \ No newline at end of file diff --git a/src/test/scala/io/viash/config_mods/PathTest.scala b/src/test/scala/io/viash/config_mods/PathTest.scala index f8d164e7d..b19ea41a4 100644 --- a/src/test/scala/io/viash/config_mods/PathTest.scala +++ b/src/test/scala/io/viash/config_mods/PathTest.scala @@ -5,8 +5,10 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ import io.circe.yaml.parser.parse +import io.viash.helpers.Logger class PathTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsing // TODO diff --git a/src/test/scala/io/viash/config_mods/PrependTest.scala b/src/test/scala/io/viash/config_mods/PrependTest.scala index 3ea413331..9a6e1335a 100644 --- a/src/test/scala/io/viash/config_mods/PrependTest.scala +++ b/src/test/scala/io/viash/config_mods/PrependTest.scala @@ -3,8 +3,10 @@ package io.viash.config_mods import io.circe.Json import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ +import io.viash.helpers.Logger class PrependSuite extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) // testing parsing test("prepend command") { val expected = ConfigMods(List( diff --git a/src/test/scala/io/viash/e2e/build/DockerMeta.scala b/src/test/scala/io/viash/e2e/build/DockerMeta.scala index 3aaf99005..b1272c09e 100644 --- a/src/test/scala/io/viash/e2e/build/DockerMeta.scala +++ b/src/test/scala/io/viash/e2e/build/DockerMeta.scala @@ -5,7 +5,7 @@ import io.viash._ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.config.Config @@ -14,6 +14,7 @@ import cats.instances.function import io.viash.functionality.resources.PlainFile class DockerMeta extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala b/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala index e5630583d..6c0bc1cea 100644 --- a/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala +++ b/src/test/scala/io/viash/e2e/build/DockerMoreSuite.scala @@ -5,13 +5,14 @@ import io.viash._ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.config.Config import scala.io.Source class DockerMoreSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/e2e/build/DockerSetup.scala b/src/test/scala/io/viash/e2e/build/DockerSetup.scala index 522ab0a78..3b2afb3ec 100644 --- a/src/test/scala/io/viash/e2e/build/DockerSetup.scala +++ b/src/test/scala/io/viash/e2e/build/DockerSetup.scala @@ -5,7 +5,7 @@ import io.viash._ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.config.Config @@ -13,6 +13,7 @@ import scala.io.Source import io.viash.functionality.resources.PlainFile class DockerSetup extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/e2e/build/DockerSuite.scala b/src/test/scala/io/viash/e2e/build/DockerSuite.scala index f3eb7af54..bb05704e2 100644 --- a/src/test/scala/io/viash/e2e/build/DockerSuite.scala +++ b/src/test/scala/io/viash/e2e/build/DockerSuite.scala @@ -5,13 +5,14 @@ import io.viash._ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import io.viash.config.Config import scala.io.Source class DockerSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/e2e/build/NativeSuite.scala b/src/test/scala/io/viash/e2e/build/NativeSuite.scala index 8af3b7994..c815c7874 100644 --- a/src/test/scala/io/viash/e2e/build/NativeSuite.scala +++ b/src/test/scala/io/viash/e2e/build/NativeSuite.scala @@ -9,17 +9,20 @@ import java.nio.file.Paths import io.viash.config.Config import scala.io.Source -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath - private val configNoPlatformFile = getClass.getResource(s"/testbash/config_no_platform.vsh.yaml").getPath private val configDeprecatedArgumentGroups = getClass.getResource(s"/testbash/config_deprecated_argument_groups.vsh.yaml").getPath private val temporaryFolder = IO.makeTemp("viash_tester") private val tempFolStr = temporaryFolder.toString + private val temporaryConfigFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + private val configDeriver = ConfigDeriver(Paths.get(configFile), temporaryConfigFolder) + // parse functionality from file private val functionality = Config.read(configFile).functionality @@ -194,10 +197,11 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { } test("when -p is omitted, the system should run as native") { + val newConfigFilePath = configDeriver.derive("""del(.platforms)""", "no_platform") val testText = TestHelper.testMain( "build", "-o", tempFolStr, - configNoPlatformFile + newConfigFilePath ) assert(executable.exists) @@ -227,7 +231,19 @@ class NativeSuite extends AnyFunSuite with BeforeAndAfterAll { assert(testRegex.findFirstIn(testOutput.error).isDefined, testOutput.error) } + test("Test config without a main script") { + val testOutput = TestHelper.testMain( + "build", + "-o", tempFolStr, + configFile, + "-c", ".functionality.resources := []" + ) + + assert(testOutput.contains("Warning: no resources specified!")) + } + override def afterAll(): Unit = { IO.deleteRecursively(temporaryFolder) + IO.deleteRecursively(temporaryConfigFolder) } } \ No newline at end of file diff --git a/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala b/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala index a4126ec71..3d7f0f7ce 100644 --- a/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala +++ b/src/test/scala/io/viash/e2e/config_inject/MainConfigInjectSuite.scala @@ -9,9 +9,10 @@ import java.nio.file.{Files, Paths, StandardCopyOption} import io.viash.config.Config import scala.io.Source -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} class MainConfigInjectSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) private val temporaryFolder = IO.makeTemp("viash_tester") val srcPath = Paths.get(getClass.getResource(s"/test_languages/").getPath()) @@ -69,13 +70,13 @@ class MainConfigInjectSuite extends AnyFunSuite with BeforeAndAfterAll { // meta_config='/tmp/viash_inject_test_languages15183647856847957861/.config.vsh.yaml' // assume number is a Long, so between 1 and 20 decimal characters - val code_replacements = code.replaceAll("inject_test_languages\\d*", "") - val code2_replacements = code2.replaceAll("inject_test_languages\\d*", "") + val code_replacements = code.replaceAll(s"inject_test_languages_$name\\d*", "") + val code2_replacements = code2.replaceAll(s"inject_test_languages_$name\\d*", "") - assert(code.length - code_replacements.length <= 126, "Stripping the paths should not cause very big differences") - assert(code.length - code_replacements.length >= 66, "Stripping the paths should not cause very big differences, but at least some") - assert(code2.length - code2_replacements.length <= 126, "Stripping the paths should not cause very big differences") - assert(code2.length - code2_replacements.length >= 66, "Stripping the paths should not cause very big differences, but at least some") + assert(code.length - code_replacements.length <= 126 + (name.length+1)*3, "Stripping the paths should not cause very big differences") + assert(code.length - code_replacements.length >= 66 + (name.length+1)*3, "Stripping the paths should not cause very big differences, but at least some") + assert(code2.length - code2_replacements.length <= 126 + (name.length+1)*3, "Stripping the paths should not cause very big differences") + assert(code2.length - code2_replacements.length >= 66 + (name.length+1)*3, "Stripping the paths should not cause very big differences, but at least some") assert(code_replacements.length == code2_replacements.length, "Running config inject multiple times should not result in substantial code differences. Only the placeholder folder is different.") } diff --git a/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala b/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala index f0a3d0f99..88d4f3f94 100644 --- a/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala +++ b/src/test/scala/io/viash/e2e/config_view/MainConfigViewSuite.scala @@ -1,10 +1,12 @@ package io.viash.e2e.config_view import io.viash._ +import io.viash.helpers.Logger import org.scalatest.funsuite.AnyFunSuite class MainConfigViewSuite extends AnyFunSuite{ + Logger.UseColorOverride.value = Some(false) // path to namespace components private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/e2e/export/MainExportSuite.scala b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala new file mode 100644 index 000000000..cf9f8d1f4 --- /dev/null +++ b/src/test/scala/io/viash/e2e/export/MainExportSuite.scala @@ -0,0 +1,206 @@ +package io.viash.e2e.export + +import io.viash._ +import io.viash.helpers.Logger +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.BeforeAndAfter + +import java.nio.file.{Files, Path} +import scala.io.Source + +class MainExportSuite extends AnyFunSuite with BeforeAndAfter { + Logger.UseColorOverride.value = Some(false) + var tempFile: Path = _ + + before { + tempFile = Files.createTempFile("viash_export", ".txt") + } + + after { + Files.deleteIfExists(tempFile) + } + + // These are all *very* basic tests. Practicly no validation whatsoever to check whether the output is correct or not. + + test("viash export resource") { + val stdout = TestHelper.testMain( + "export", "resource", "platforms/nextflow/WorkflowHelper.nf" + ) + + assert(stdout.startsWith("/////////////////////////////////////\n// Viash Workflow helper functions //")) + assert(stdout.contains("preprocessInputs")) + } + + test("viash export resource to file") { + val stdout = TestHelper.testMain( + "export", "resource", "platforms/nextflow/WorkflowHelper.nf", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("/////////////////////////////////////\n// Viash Workflow helper functions //")) + assert(lines.contains("preprocessInputs")) + } + + test("viash export cli_schema") { + val stdout = TestHelper.testMain( + "export", "cli_schema" + ) + + assert(stdout.startsWith("""- name: "run"""")) + assert(stdout.contains("viash config inject")) + } + + test("viash export cli_schema to file") { + val stdout = TestHelper.testMain( + "export", "cli_schema", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("""- name: "run"""")) + assert(lines.contains("viash config inject")) + } + + test("viash export cli_autocomplete without format") { + val stdout = TestHelper.testMain( + "export", "cli_autocomplete" + ) + + assert(stdout.startsWith("""# bash completion for viash""")) + assert(stdout.contains("COMPREPLY=($(compgen -W 'run build test ns config' -- \"$cur\"))")) + } + + test("viash export cli_autocomplete without format to file") { + val stdout = TestHelper.testMain( + "export", "cli_autocomplete", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("""# bash completion for viash""")) + assert(lines.contains("COMPREPLY=($(compgen -W")) + } + + test("viash export cli_autocomplete Bash") { + val stdout = TestHelper.testMain( + "export", "cli_autocomplete", + "--format", "bash" + ) + + assert(stdout.startsWith("""# bash completion for viash""")) + assert(stdout.contains("COMPREPLY=($(compgen -W 'run build test ns config' -- \"$cur\"))")) + } + + test("viash export cli_autocomplete Bash to file") { + val stdout = TestHelper.testMain( + "export", "cli_autocomplete", + "--format", "bash", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("""# bash completion for viash""")) + assert(lines.contains("COMPREPLY=($(compgen -W")) + } + + test("viash export cli_autocomplete Zsh") { + val stdout = TestHelper.testMain( + "export", "cli_autocomplete", + "--format", "zsh" + ) + + assert(stdout.startsWith("""#compdef viash""")) + assert(stdout.contains("_viash_export_commands")) + } + + test("viash export cli_autocomplete Zsh to file") { + val stdout = TestHelper.testMain( + "export", "cli_autocomplete", + "--format", "zsh", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("""#compdef viash""")) + assert(lines.contains("_viash_export_commands")) + } + + test("viash export config_schema") { + val stdout = TestHelper.testMain( + "export", "config_schema" + ) + + assert(stdout.startsWith("""- - name: "__this__"""")) + assert(stdout.contains("""type: "OneOrMore[String]"""")) + } + + test("viash export config_schema to file") { + val stdout = TestHelper.testMain( + "export", "config_schema", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("""- - name: "__this__"""")) + assert(lines.contains("""type: "OneOrMore[String]"""")) + } + + test("viash export json_schema") { + val stdout = TestHelper.testMain( + "export", "json_schema" + ) + + assert(stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#"""")) + assert(stdout.contains("""- $ref: "#/definitions/Config"""")) + } + + test("viash export json_schema, explicit yaml format") { + val stdout = TestHelper.testMain( + "export", "json_schema", "--format", "yaml" + ) + + assert(stdout.startsWith("""$schema: "https://json-schema.org/draft-07/schema#"""")) + assert(stdout.contains("""- $ref: "#/definitions/Config"""")) + } + + test("viash export json_schema to file, explicit yaml format") { + val stdout = TestHelper.testMain( + "export", "json_schema", "--format", "yaml", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith("""$schema: "https://json-schema.org/draft-07/schema#"""")) + assert(lines.contains("""- $ref: "#/definitions/Config"""")) + } + + test("viash export json_schema, json format") { + val stdout = TestHelper.testMain( + "export", "json_schema", "--format", "json" + ) + + assert(stdout.startsWith( + """{ + | "$schema" : "https://json-schema.org/draft-07/schema#", + | "definitions" : { + |""".stripMargin)) + assert(stdout.contains(""""$ref" : "#/definitions/Config"""")) + } + + test("viash export json_schema to file, json format") { + val stdout = TestHelper.testMain( + "export", "json_schema", "--format", "json", + "--output", tempFile.toString + ) + + val lines = helpers.IO.read(tempFile.toUri()) + assert(lines.startsWith( + """{ + | "$schema" : "https://json-schema.org/draft-07/schema#", + | "definitions" : { + |""".stripMargin)) + assert(lines.contains(""""$ref" : "#/definitions/Config"""")) + } + +} diff --git a/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala b/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala new file mode 100644 index 000000000..0bc286db2 --- /dev/null +++ b/src/test/scala/io/viash/e2e/help/MainHelpSuite.scala @@ -0,0 +1,79 @@ +package io.viash.e2e.help + +import io.viash._ +import io.viash.helpers.Logger +import org.scalatest.funsuite.AnyFunSuite +import io.viash.exceptions.ExitException + +class MainHelpSuite extends AnyFunSuite{ + Logger.UseColorOverride.value = Some(false) + // path to namespace components + private val configFile = getClass.getResource(s"/testbash/config.vsh.yaml").getPath + + test("viash config view default functionality without help") { + val stdout = TestHelper.testMain( + "config", "view", + configFile + ) + + assert(stdout.startsWith("functionality:")) + assert(stdout.contains("testbash")) + } + + test("viash config view default functionality leading help") { + val output = TestHelper.testMainException[ExitException]( + "config", "view", + "--help" + ) + + assert(output.startsWith("viash config view")) + assert(!output.contains("testbash")) + } + + test("viash config view default functionality trailing help") { + val output = TestHelper.testMainException[ExitException]( + "config", "view", + configFile, + "--help" + ) + + assert(output.startsWith("viash config view")) + assert(!output.contains("testbash")) + } + + test("viash config view default functionality trailing help after platform argument") { + val output = TestHelper.testMainException[ExitException]( + "config", "view", + configFile, + "--platform", "native", + "--help" + ) + + assert(output.startsWith("viash config view")) + assert(!output.contains("testbash")) + } + + test("viash config view default functionality trailing help before platform argument") { + val output = TestHelper.testMainException[ExitException]( + "config", "view", + configFile, + "--help", + "--platform", "native" + ) + + assert(output.startsWith("viash config view")) + assert(!output.contains("testbash")) + } + + + test("viash config view default functionality with --help as platform argument") { + val output = TestHelper.testMainException[RuntimeException]( + "config", "view", + configFile, + "--platform", "--help" + ) + + assert(!output.contains("viash config view")) + } + +} diff --git a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala index d62bad29e..269d7ffcd 100644 --- a/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_build/MainNSBuildNativeSuite.scala @@ -3,7 +3,7 @@ package io.viash.e2e.ns_build import io.viash._ import io.viash.config.Config -import io.viash.helpers.{Exec, IO} +import io.viash.helpers.{Exec, IO, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -12,6 +12,7 @@ import java.nio.file.Paths import scala.io.Source class MainNSBuildNativeSuite extends AnyFunSuite with BeforeAndAfterAll{ + Logger.UseColorOverride.value = Some(false) // path to namespace components private val nsPath = getClass.getResource("/testns/").getPath diff --git a/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala index 0d9f8c0af..e7275fa04 100644 --- a/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_exec/MainNSExecNativeSuite.scala @@ -2,7 +2,7 @@ package io.viash.e2e.ns_exec import io.viash._ -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -11,6 +11,7 @@ import java.nio.file.{Files, OpenOption, Paths} import scala.io.Source class MainNSExecNativeSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // path to namespace components private val nsPath = getClass.getResource("/testns/src/").getPath diff --git a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala index 9000234da..59fc8afe1 100644 --- a/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_list/MainNSListNativeSuite.scala @@ -3,7 +3,7 @@ package io.viash.e2e.ns_list import io.viash._ import io.viash.config.Config -import io.viash.helpers.{Exec, IO} +import io.viash.helpers.{Exec, IO, Logger} import io.circe.yaml.parser import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -13,6 +13,7 @@ import java.nio.file.Paths import scala.io.Source class MainNSListNativeSuite extends AnyFunSuite{ + Logger.UseColorOverride.value = Some(false) // path to namespace components private val nsPath = getClass.getResource("/testns/").getPath private val scalaPath = getClass.getResource("/test_languages/scala/").getPath @@ -28,8 +29,8 @@ class MainNSListNativeSuite extends AnyFunSuite{ // convert testbash test("viash ns list") { - val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( - "ns", "list", + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", "-s", nsPath, ) @@ -67,7 +68,7 @@ class MainNSListNativeSuite extends AnyFunSuite{ // convert testbash test("viash ns list filter by platform") { val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( - "ns", "list", + "ns", "list", "-s", nsPath, "-p", "docker" ) @@ -80,7 +81,7 @@ class MainNSListNativeSuite extends AnyFunSuite{ } test("viash ns list filter by platform #2") { val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( - "ns", "list", + "ns", "list", "-s", scalaPath, "-p", "docker" ) @@ -94,7 +95,7 @@ class MainNSListNativeSuite extends AnyFunSuite{ } test("viash ns list filter by platform #3") { val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( - "ns", "list", + "ns", "list", "-s", scalaPath, "-p", "not_exists" ) @@ -106,4 +107,202 @@ class MainNSListNativeSuite extends AnyFunSuite{ assert(configs.length == 0) } + // test query_name + test("viash ns list query_name") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_name", "ns_add" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query_name full match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_name", "^ns_add$" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query_name partial match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_name", "add" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query_name no match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_name", "foo" + ) + + assert(exitCode == 1) + assert(stdout.trim() == "[]") + } + + // test query + test("viash ns list query") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "-q", "testns/ns_add" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query full match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "-q", "^testns/ns_add$" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query partial match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "-q", "test.*/.*add" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query only partial name") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "-q", "add" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + assert(configs.length == 1) + assert(stdout.contains("name: \"ns_add\"")) + } + + test("viash ns list query no match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "-q", "foo" + ) + + assert(exitCode == 1) + assert(stdout.trim() == "[]") + } + + // test query_namespace + test("viash ns list query_namespace") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_namespace", "testns" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + + assert(configs.length == components.length) + assert(stdout.contains("name: \"ns_add\"")) + assert(stdout.contains("name: \"ns_subtract\"")) + assert(!stdout.contains("name: \"ns_error\"")) + assert(!stdout.contains("name: \"ns_disabled\"")) + } + + test("viash ns list query_namespace full match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_namespace", "^testns$" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + + assert(configs.length == components.length) + assert(stdout.contains("name: \"ns_add\"")) + assert(stdout.contains("name: \"ns_subtract\"")) + assert(!stdout.contains("name: \"ns_error\"")) + assert(!stdout.contains("name: \"ns_disabled\"")) + } + + test("viash ns list query_namespace partial match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_namespace", "test" + ) + + assert(exitCode == 1) + val configs = parser.parse(stdout) + .fold(throw _, _.as[Array[Config]]) + .fold(throw _, identity) + + assert(configs.length == components.length) + assert(stdout.contains("name: \"ns_add\"")) + assert(stdout.contains("name: \"ns_subtract\"")) + assert(!stdout.contains("name: \"ns_error\"")) + assert(!stdout.contains("name: \"ns_disabled\"")) + } + + test("viash ns list query_namespace no match") { + val (stdout, stderr, exitCode) = TestHelper.testMainWithStdErr( + "ns", "list", + "-s", nsPath, + "--query_namespace", "foo" + ) + + assert(exitCode == 1) + assert(stdout.trim() == "[]") + } + } diff --git a/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala b/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala index 9bcd05515..489ab1ec2 100644 --- a/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/ns_test/MainNSTestNativeSuite.scala @@ -2,7 +2,7 @@ package io.viash.e2e.ns_test import io.viash._ -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -11,6 +11,7 @@ import java.nio.file.{Files, OpenOption, Paths} import scala.io.Source class MainNSTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // path to namespace components private val nsPath = getClass.getResource("/testns/").getPath diff --git a/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala new file mode 100644 index 000000000..ebebe2028 --- /dev/null +++ b/src/test/scala/io/viash/e2e/run/MainRunDockerSuite.scala @@ -0,0 +1,92 @@ +package io.viash.e2e.run + +import io.viash._ + +import java.nio.file.{Files, Paths, StandardCopyOption} + +import io.viash.helpers.{IO, Exec} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite + +import scala.reflect.io.Directory +import sys.process._ + +class MainRunDockerSuite extends AnyFunSuite with BeforeAndAfterAll { + private val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") + private val temporaryFolder2 = temporaryFolder.resolve("folder with spaces") + + private val configText = + """functionality: + | name: testing + | arguments: + | - type: file + | name: --input + | required: true + | - type: file + | direction: output + | name: --output + | required: true + | resources: + | - type: bash_script + | text: | + | cp -r "$par_input" "$par_output" + |platforms: + | - type: docker + | image: python:3.10-slim + |""".stripMargin + + test("Check run with standard input and output folders", DockerTest) { + val configFile = temporaryFolder.resolve("config.vsh.yaml") + Files.write(configFile, configText.getBytes()) + + val inputFile = temporaryFolder.resolve("some_input") + Files.write(inputFile, "foo".getBytes()) + + val outputFile = temporaryFolder.resolve("some_output") + + val runText = TestHelper.testMain( + "run", + configFile.toString(), + "--", + "--input", inputFile.toString(), + "--output", outputFile.toString() + ) + + // assert(runText == "", "expecting the output to be empty") + + val outputFileText = Files.readString(outputFile) + + assert(outputFileText == "foo") + } + + test("Check run with a folder containing a space", DockerTest) { + + temporaryFolder2.toFile.mkdir() + + val configFile = temporaryFolder2.resolve("config.vsh.yaml") + Files.write(configFile, configText.getBytes()) + + val inputFile = temporaryFolder2.resolve("some_input") + Files.write(inputFile, "bar".getBytes()) + + val outputFile = temporaryFolder2.resolve("some_output") + + val runText = TestHelper.testMain( + "run", + configFile.toString(), + "--", + "--input", inputFile.toString(), + "--output", outputFile.toString() + ) + + // assert(runText == "", "expecting the output to be empty") + + val outputFileText = Files.readString(outputFile) + + assert(outputFileText == "bar") + } + + override def afterAll(): Unit = { + IO.deleteRecursively(temporaryFolder) + } +} diff --git a/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala b/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala index 82e856a33..768ca2dbf 100644 --- a/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala +++ b/src/test/scala/io/viash/e2e/run/RunComputationalRequirements.scala @@ -1,13 +1,13 @@ package io.viash.e2e.run import io.viash.{ConfigDeriver, TestHelper} -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import java.nio.file.Paths import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite class RunComputationalRequirements extends AnyFunSuite with BeforeAndAfterAll { - + Logger.UseColorOverride.value = Some(false) private val configFile = getClass.getResource("/testbash/check_computational_requirements.vsh.yaml").getPath private val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") private val tempFolStr = temporaryFolder.toString diff --git a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala index 1d9674183..191d32eeb 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala @@ -4,14 +4,16 @@ import io.viash._ import java.nio.file.{Files, Paths, StandardCopyOption} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import scala.reflect.io.Directory import sys.process._ +import org.scalatest.ParallelTestExecution -class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll { +class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with ParallelTestExecution{ + Logger.UseColorOverride.value = Some(false) // default yaml private val configFile = getClass.getResource("/testbash/config.vsh.yaml").getPath @@ -64,6 +66,42 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll { checkTempDirAndRemove(testText, false) } + test("Check setup strategy", DockerTest) { + // first run to create cache entries + val testText = TestHelper.testMain( + "test", + "-p", "docker", + configFile, + "--keep", "false" + ) + + // Do a second run to check if forcing a docker build using setup works + val testTextNoCaching = TestHelper.testMain( + "test", + "-p", "docker", + configFile, + "--setup", "build", + "--keep", "false" + ) + + val regexBuildCache = raw"RUN.*:\n.*CACHED".r + assert(!regexBuildCache.findFirstIn(testTextNoCaching).isDefined, "Expected to not find caching.") + + // Do a third run to check caching + val testTextCaching = TestHelper.testMain( + "test", + "-p", "docker", + configFile, + "--setup", "cb", + "--keep", "false" + ) + assert(regexBuildCache.findFirstIn(testTextCaching).isDefined, "Expected to find caching.") + checkTempDirAndRemove(testText, false) + checkTempDirAndRemove(testTextCaching, false) + checkTempDirAndRemove(testTextNoCaching, false) + + } + test("Verify base config derivation", NativeTest) { val newConfigFilePath = configDeriver.derive(Nil, "default_config") val testText = TestHelper.testMain( diff --git a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala index 770d8dd59..0558ccff1 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestNativeSuite.scala @@ -4,7 +4,7 @@ import io.viash._ import java.nio.file.{Files, Paths, StandardCopyOption} -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -12,6 +12,7 @@ import scala.reflect.io.Directory import sys.process._ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // default yaml private val configFile = getClass.getResource("/testbash/config.vsh.yaml").getPath @@ -143,30 +144,48 @@ class MainTestNativeSuite extends AnyFunSuite with BeforeAndAfterAll { checkTempDirAndRemove(testText, false) } - test("Check standard test output with legacy 'tests' definition") { - val newConfigFilePath = configDeriver.derive( - List( - """.functionality.tests := .functionality.test_resources""", - """del(.functionality.test_resources)""" - ) , "legacy") - val testOutput = TestHelper.testMainException2[Exception]( + test("Check config file without 'functionality' specified") { + val newConfigFilePath = configDeriver.derive("""del(.functionality)""", "missing_functionality") + val testOutput = TestHelper.testMainException2[RuntimeException]( "test", "-p", "native", newConfigFilePath ) - assert(testOutput.error.contains("Error: .functionality.tests was removed: Use `test_resources` instead. No functional difference. Initially deprecated 0.5.13, removed 0.7.0.")) + assert(testOutput.exceptionText.contains("must be a yaml file containing a viash config.")) + assert(testOutput.output.isEmpty) } - test("Check config file without 'functionality' specified") { - val newConfigFilePath = configDeriver.derive("""del(.functionality)""", "missing_functionality") + test("Check invalid platform type") { + val newConfigFilePath = configDeriver.derive(""".platforms += { type: "foo" }""", "invalid_platform_type") val testOutput = TestHelper.testMainException2[RuntimeException]( "test", "-p", "native", newConfigFilePath ) - assert(testOutput.exceptionText.contains("must be a yaml file containing a viash config.")) + assert(testOutput.exceptionText.contains("Type 'foo' is not recognised. Valid types are 'docker', 'native', and 'nextflow'.")) + assert(testOutput.exceptionText.contains( + """{ + | "type" : "foo" + |}""".stripMargin)) + assert(testOutput.output.isEmpty) + } + + test("Check invalid field in platform") { + val newConfigFilePath = configDeriver.derive(""".platforms += { type: "native", foo: "bar" }""", "invalid_platform_field") + val testOutput = TestHelper.testMainException2[RuntimeException]( + "test", + "-p", "native", + newConfigFilePath + ) + + assert(testOutput.exceptionText.contains("Invalid data fields for NativePlatform.")) + assert(testOutput.exceptionText.contains( + """{ + | "type" : "native", + | "foo" : "bar" + |}""".stripMargin)) assert(testOutput.output.isEmpty) } diff --git a/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala b/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala index e9aa3e180..b8a7f48cc 100644 --- a/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala +++ b/src/test/scala/io/viash/e2e/test/TestComputationalRequirements.scala @@ -1,13 +1,13 @@ package io.viash.e2e.test import io.viash.{ConfigDeriver, TestHelper} -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import java.nio.file.Paths import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite class TestComputationalRequirements extends AnyFunSuite with BeforeAndAfterAll { - + Logger.UseColorOverride.value = Some(false) private val configFile = getClass.getResource("/testbash/check_computational_requirements.vsh.yaml").getPath private val temporaryFolder = IO.makeTemp(s"viash_${this.getClass.getName}_") private val tempFolStr = temporaryFolder.toString diff --git a/src/test/scala/io/viash/escaping/EscapingNativeTest.scala b/src/test/scala/io/viash/escaping/EscapingNativeTest.scala index 0e1897d6c..48489c6e5 100644 --- a/src/test/scala/io/viash/escaping/EscapingNativeTest.scala +++ b/src/test/scala/io/viash/escaping/EscapingNativeTest.scala @@ -8,9 +8,10 @@ import org.scalatest.funsuite.AnyFunSuite import java.io.{IOException, UncheckedIOException} import java.nio.file.{Files, Path, Paths} import scala.io.Source -import io.viash.helpers.{IO, Exec} +import io.viash.helpers.{IO, Exec, Logger} class EscapingNativeTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // which platform to test private val rootPath = getClass.getResource(s"/test_escaping/").getPath private val configFile = getClass.getResource(s"/test_escaping/config.vsh.yaml").getPath diff --git a/src/test/scala/io/viash/functionality/FunctionalityTest.scala b/src/test/scala/io/viash/functionality/FunctionalityTest.scala index ae81d188b..afb491d86 100644 --- a/src/test/scala/io/viash/functionality/FunctionalityTest.scala +++ b/src/test/scala/io/viash/functionality/FunctionalityTest.scala @@ -9,9 +9,10 @@ import io.circe.yaml.{parser => YamlParser} import io.circe.syntax._ import io.viash.helpers.circe._ import io.viash.helpers.data_structures._ - +import io.viash.helpers.Logger class FunctionalityTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) val infoJson = Yaml(""" |foo: | bar: diff --git a/src/test/scala/io/viash/functionality/arguments/DoubleInfinityTest.scala b/src/test/scala/io/viash/functionality/arguments/DoubleInfinityTest.scala new file mode 100644 index 000000000..db0eed8f5 --- /dev/null +++ b/src/test/scala/io/viash/functionality/arguments/DoubleInfinityTest.scala @@ -0,0 +1,139 @@ +package io.viash.functionality.arguments + +import org.scalatest.funsuite.AnyFunSuite +import io.circe._ + +import io.viash.helpers.Logger +import io.viash.functionality.arguments.DoubleArgument +import io.viash.functionality._ +import io.viash.helpers.Yaml +import io.viash.helpers.circe.Convert +import io.viash.functionality.arguments.encodeArgument +import io.circe.yaml.Printer + +class DoubleInfinityTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) + + test("Regular double values are parsed correctly") { + val inputText = + """ + |name: "--double" + |min: 5 + |""".stripMargin + val yaml = Yaml.replaceInfinities(inputText) + val yaml2 = Convert.textToJson(inputText, "foo") + val arg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(arg.min == Some(5)) + } + + test(".nan values are parsed correctly") { + val inputText = + """ + |name: "--double" + |min: ".nan" + |""".stripMargin + val yaml = Yaml.replaceInfinities(inputText) + val yaml2 = Convert.textToJson(yaml, "foo") + val arg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(arg.min.isDefined) + assert(arg.min.get.isNaN) + } + + test("+.inf values are parsed correctly") { + val inputText = + """ + |name: "--double" + |min: +.inf + |""".stripMargin + val yaml = Yaml.replaceInfinities(inputText) + val yaml2 = Convert.textToJson(yaml, "foo") + val arg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(arg.min.isDefined) + assert(arg.min.get.isPosInfinity) + } + + test("-.inf values are parsed correctly") { + val inputText = + """ + |name: "--double" + |min: -.inf + |""".stripMargin + val yaml = Yaml.replaceInfinities(inputText) + val yaml2 = Convert.textToJson(yaml, "foo") + val arg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(arg.min.isDefined) + assert(arg.min.get.isNegInfinity) + } + + test("Regular double values can be serialized and parsed back") { + val inputArg = DoubleArgument( + name = "--double", + min = Some(5) + ) + val json = encodeArgument(inputArg) + val yaml = Printer.spaces2.pretty(json) + + assert(yaml.contains("min: 5.0")) + + val yaml2 = Convert.textToJson(yaml, "foo") + val outputArg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(outputArg.min == Some(5)) + } + + test(".nan values can be serialized and parsed back") { + val inputArg = DoubleArgument( + name = "--double", + min = Some(Double.NaN) + ) + val json = encodeArgument(inputArg) + val yaml = Printer.spaces2.pretty(json) + + assert(yaml.contains("min: NaN")) + + val yaml2 = Convert.textToJson(yaml, "foo") + val outputArg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(outputArg.min.isDefined) + assert(outputArg.min.get.isNaN) + } + + test("+.inf values can be serialized and parsed back") { + val inputArg = DoubleArgument( + name = "--double", + min = Some(Double.PositiveInfinity) + ) + val json = encodeArgument(inputArg) + val yaml = Printer.spaces2.pretty(json) + + assert(yaml.contains("min: +Infinity")) + + val yaml2 = Convert.textToJson(yaml, "foo") + val outputArg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(outputArg.min.isDefined) + assert(outputArg.min.get.isPosInfinity) + } + + test("-.inf values can be serialized and parsed back") { + val inputArg = DoubleArgument( + name = "--double", + min = Some(Double.NegativeInfinity) + ) + val json = encodeArgument(inputArg) + val yaml = Printer.spaces2.pretty(json) + + assert(yaml.contains("min: -Infinity")) + + val yaml2 = Convert.textToJson(yaml, "foo") + val outputArg = Convert.jsonToClass[DoubleArgument](yaml2, "foo") + + assert(outputArg.min.isDefined) + assert(outputArg.min.get.isNegInfinity) + } + +} \ No newline at end of file diff --git a/src/test/scala/io/viash/functionality/arguments/StringArgumentTest.scala b/src/test/scala/io/viash/functionality/arguments/StringArgumentTest.scala index 7e21c0c39..92435a872 100644 --- a/src/test/scala/io/viash/functionality/arguments/StringArgumentTest.scala +++ b/src/test/scala/io/viash/functionality/arguments/StringArgumentTest.scala @@ -9,9 +9,10 @@ import io.circe.syntax._ import io.circe.yaml.{parser => YamlParser} import io.viash.helpers.circe._ import io.viash.helpers.data_structures._ - +import io.viash.helpers.Logger class StringArgumentTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) val infoJson = Yaml(""" |foo: | bar: diff --git a/src/test/scala/io/viash/functionality/resources/StringArgumentTest.scala b/src/test/scala/io/viash/functionality/resources/StringArgumentTest.scala new file mode 100644 index 000000000..274f9ff0d --- /dev/null +++ b/src/test/scala/io/viash/functionality/resources/StringArgumentTest.scala @@ -0,0 +1,90 @@ +package io.viash.functionality.resources + +import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger + +class CommandsTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) + + test("Test Bash script") { + val script = BashScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "bash \"foo\"") + assert(commandSeq == Seq("bash", "foo")) + } + + test("Test CSharp script") { + val script = CSharpScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "dotnet script \"foo\"") + assert(commandSeq == Seq("dotnet", "script", "foo")) + } + + test("Test Executable") { + val script = Executable(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "foo") + assert(commandSeq == Seq("foo")) + } + + test("Test JavaScript script") { + val script = JavaScriptScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "node \"foo\"") + assert(commandSeq == Seq("node", "foo")) + } + + test("Test Nextflow script") { + val script = NextflowScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "nextflow run . -main-script \"foo\"") + assert(commandSeq == Seq("nextflow", "run", ".", "-main-script", "foo")) + } + + test("Test Nextflow script with entrypoint") { + val script = NextflowScript(path = Some("bar"), entrypoint = Some("baz")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "nextflow run . -main-script \"foo\" -entry baz") + assert(commandSeq == Seq("nextflow", "run", ".", "-main-script", "foo", "-entry", "baz")) + } + + test("Test Python script") { + val script = PythonScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "python -B \"foo\"") + assert(commandSeq == Seq("python", "-B", "foo")) + } + + test ("Test R script") { + val script = RScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "Rscript \"foo\"") + assert(commandSeq == Seq("Rscript", "foo")) + } + + test("Test Scala script") { + val script = ScalaScript(path = Some("bar")) + val command = script.command("foo") + val commandSeq = script.commandSeq("foo") + + assert(command == "scala -nc \"foo\"") + assert(commandSeq == Seq("scala", "-nc", "foo")) + } + +} \ No newline at end of file diff --git a/src/test/scala/io/viash/helpers/EscaperTest.scala b/src/test/scala/io/viash/helpers/EscaperTest.scala index 784fdfe96..be1458545 100644 --- a/src/test/scala/io/viash/helpers/EscaperTest.scala +++ b/src/test/scala/io/viash/helpers/EscaperTest.scala @@ -2,8 +2,10 @@ package io.viash.helpers import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger class EscaperTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) val s = "a \\ b $ c ` d \" e ' f \n g" test("escape with default parameters work") { diff --git a/src/test/scala/io/viash/helpers/ExecTest.scala b/src/test/scala/io/viash/helpers/ExecTest.scala index 3b7836dad..ff6f37c88 100644 --- a/src/test/scala/io/viash/helpers/ExecTest.scala +++ b/src/test/scala/io/viash/helpers/ExecTest.scala @@ -4,8 +4,10 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} import scala.util.Try +import io.viash.helpers.Logger class ExecTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) test("Check exec.run") { val execRun = Exec.run(List("echo", "hi")) assert(execRun.trim == "hi") diff --git a/src/test/scala/io/viash/helpers/FormatTest.scala b/src/test/scala/io/viash/helpers/FormatTest.scala new file mode 100644 index 000000000..901d294dd --- /dev/null +++ b/src/test/scala/io/viash/helpers/FormatTest.scala @@ -0,0 +1,86 @@ +package io.viash.helpers + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger + +class FormatTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + test("A paragraph that contains no newlines that is longer than the wrap column is wrapped at the wrap column") { + val paragraph = "A paragraph that contains no newlines that is longer than the wrap column is wrapped at the wrap column" + val wrapColumn = 80 + val expected = List( + "A paragraph that contains no newlines that is longer than the wrap column is", + "wrapped at the wrap column" + ) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } + + test("A paragraph that contains no newlines that is shorter than the wrap column is not wrapped") { + val paragraph = "A paragraph that contains no newlines that is shorter than the wrap column" + val wrapColumn = 80 + val expected = List(paragraph) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } + + test("A paragraph that contains a newline is split at the newline") { + val paragraph = "A paragraph that contains a newline\nand is longer than the wrap column" + val wrapColumn = 80 + val expected = List( + "A paragraph that contains a newline", + "and is longer than the wrap column" + ) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } + + test("A paragraph that contains a newline and is longer than the wrap column wraps at the wrap column after the newline") { + val paragraph = "A paragraph that contains a newline\nand is longer than the wrap column after the newline" + val wrapColumn = 40 + val expected = List( + "A paragraph that contains a newline", + "and is longer than the wrap column after", + "the newline" + ) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } + + test("A paragraph that contains a newline and is shorter than the wrap column is not wrapped") { + val paragraph = "A paragraph that contains a newline\nand is shorter than the wrap column" + val wrapColumn = 80 + val expected = List( + "A paragraph that contains a newline", + "and is shorter than the wrap column" + ) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } + + test("A paragraph that contains multiple newlines is split at each newline") { + val paragraph = "A paragraph that contains multiple newlines\nand is longer than the wrap column\nafter each newline" + val wrapColumn = 80 + val expected = List( + "A paragraph that contains multiple newlines", + "and is longer than the wrap column", + "after each newline" + ) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } + + test("A paragraph that contains multiple newlines and is longer than the wrap column wraps at the wrap column after each newline") { + val paragraph = "A paragraph that contains multiple newlines\nand is longer than the wrap column\nafter each newline" + val wrapColumn = 40 + val expected = List( + "A paragraph that contains multiple", + "newlines", + "and is longer than the wrap column", + "after each newline" + ) + val actual = Format.paragraphWrap(paragraph, wrapColumn) + assert(actual == expected) + } +} diff --git a/src/test/scala/io/viash/helpers/GitTest.scala b/src/test/scala/io/viash/helpers/GitTest.scala index d37f37c7c..b7a3c9c65 100644 --- a/src/test/scala/io/viash/helpers/GitTest.scala +++ b/src/test/scala/io/viash/helpers/GitTest.scala @@ -5,8 +5,10 @@ import org.scalatest.funsuite.AnyFunSuite import java.nio.file.{Files, Paths, StandardCopyOption} import java.nio.file.Path import scala.collection.mutable.ListBuffer +import io.viash.helpers.Logger class GitTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) val fakeGitRepo = "git@non.existing.repo:viash/meta-test" val tempPaths = ListBuffer[Path]() diff --git a/src/test/scala/io/viash/helpers/IOTest.scala b/src/test/scala/io/viash/helpers/IOTest.scala index 2f6eeed09..97bde729c 100644 --- a/src/test/scala/io/viash/helpers/IOTest.scala +++ b/src/test/scala/io/viash/helpers/IOTest.scala @@ -4,10 +4,11 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.BeforeAndAfter import java.nio.file.{Files, Path} import java.net.URI -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import scala.util.Try class IOTest extends AnyFunSuite with BeforeAndAfter { + Logger.UseColorOverride.value = Some(false) var tempDir: Path = _ before { diff --git a/src/test/scala/io/viash/helpers/LoggerTest.scala b/src/test/scala/io/viash/helpers/LoggerTest.scala new file mode 100644 index 000000000..457fd0019 --- /dev/null +++ b/src/test/scala/io/viash/helpers/LoggerTest.scala @@ -0,0 +1,422 @@ +package io.viash.helpers + +import org.scalatest.funsuite.AnyFunSuite +import java.io.ByteArrayOutputStream +import scala.io.AnsiColor +import io.viash.TestHelper +import java.io.FileNotFoundException + +class LoggerTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) + + test("Check basic log print") { + val logger = Logger.apply("Tester") + + assert(logger.name == "Tester") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.info("foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr == "foo\n") + } + + test("Check print to stdout") { + val logger = Logger.apply("Tester") + + assert(logger.name == "Tester") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.infoOut("foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout == "foo\n") + assert(stderr.isEmpty()) + } + + test("Check printing in color") { + val logger = Logger.apply("Tester_color", LoggerLevel.Info, true) + + assert(logger.name == "Tester_color") + assert(logger.useColor == true) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.info("foo") + logger.infoOut("bar") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout == s"${AnsiColor.WHITE}bar${AnsiColor.RESET}\n") + assert(stderr == s"${AnsiColor.WHITE}foo${AnsiColor.RESET}\n") + } + + test("Check printing without color") { + val logger = Logger.apply("Tester_no_color", LoggerLevel.Info, false) + + assert(logger.name == "Tester_no_color") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.info("foo") + logger.infoOut("bar") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout == "bar\n") + assert(stderr == "foo\n") + } + + test("Check error log print") { + val logger = Logger.apply("Tester_error") + + assert(logger.name == "Tester_error") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.error("foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr == "foo\n") + } + + test("Check warn log print") { + val logger = Logger.apply("Tester_warn") + + assert(logger.name == "Tester_warn") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.warn("foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr == "foo\n") + } + + test("Check debug log print while minimum level is info") { + val logger = Logger.apply("Tester_debug") + + assert(logger.name == "Tester_debug") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.debug("foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr.isEmpty()) + } + + test("Check debug log print while minimum level is debug") { + val logger = Logger.apply("Tester_debug2", LoggerLevel.Debug, false) + + assert(logger.name == "Tester_debug2") + assert(logger.useColor == false) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.debug("foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr == "foo\n") + } + + test("Check is*Enabled methods") { + val logger = Logger.apply("Tester") + + assert(logger.isErrorEnabled == true) + assert(logger.isWarnEnabled == true) + assert(logger.isInfoEnabled == true) + assert(logger.isDebugEnabled == false) + assert(logger.isTraceEnabled == false) + } + + test("Check LoggerLevel from string") { + assert(LoggerLevel.fromString("error") == LoggerLevel.Error) + assert(LoggerLevel.fromString("warn") == LoggerLevel.Warn) + assert(LoggerLevel.fromString("info") == LoggerLevel.Info) + assert(LoggerLevel.fromString("debug") == LoggerLevel.Debug) + assert(LoggerLevel.fromString("trace") == LoggerLevel.Trace) + assertThrows[RuntimeException](LoggerLevel.fromString("foo")) + } + + test("Check all level prints") { + val logger = Logger.apply("Tester_prints_color", LoggerLevel.Trace, true) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + logger.error("err: error") + logger.warn("err: warn") + logger.info("err: info") + logger.debug("err: debug") + logger.trace("err: trace") + logger.success("err: success") + + logger.errorOut("out: error") + logger.warnOut("out: warn") + logger.infoOut("out: info") + logger.debugOut("out: debug") + logger.traceOut("out: trace") + logger.successOut("out: success") + + logger.log(LoggerOutput.StdErr, LoggerLevel.Error, AnsiColor.MAGENTA, "err: foo") + logger.log(LoggerOutput.StdOut, LoggerLevel.Error, AnsiColor.BLUE, "out: foo") + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + val expectOut = + s"""${AnsiColor.RED}out: error${AnsiColor.RESET} + |${AnsiColor.YELLOW}out: warn${AnsiColor.RESET} + |${AnsiColor.WHITE}out: info${AnsiColor.RESET} + |${AnsiColor.GREEN}out: debug${AnsiColor.RESET} + |${AnsiColor.CYAN}out: trace${AnsiColor.RESET} + |${AnsiColor.GREEN}out: success${AnsiColor.RESET} + |${AnsiColor.BLUE}out: foo${AnsiColor.RESET} + |""".stripMargin + + val expectErr = + s"""${AnsiColor.RED}err: error${AnsiColor.RESET} + |${AnsiColor.YELLOW}err: warn${AnsiColor.RESET} + |${AnsiColor.WHITE}err: info${AnsiColor.RESET} + |${AnsiColor.GREEN}err: debug${AnsiColor.RESET} + |${AnsiColor.CYAN}err: trace${AnsiColor.RESET} + |${AnsiColor.GREEN}err: success${AnsiColor.RESET} + |${AnsiColor.MAGENTA}err: foo${AnsiColor.RESET} + |""".stripMargin + + assert(stdout == expectOut) + assert(stderr == expectErr) + } + + test("Check logger as a class trait") { + + class ClassTraitLoggingTest extends Logging { + def bar(): Unit = { + error(s"err: error $isErrorEnabled") + warn(s"err: warn $isWarnEnabled") + info(s"err: info $isInfoEnabled") + debug(s"err: debug $isDebugEnabled") + trace(s"err: trace $isTraceEnabled") + success(s"err: success") + + errorOut(s"out: error") + warnOut(s"out: warn") + infoOut(s"out: info") + debugOut(s"out: debug") + traceOut(s"out: trace") + successOut(s"out: success") + + log(LoggerOutput.StdErr, LoggerLevel.Error, AnsiColor.MAGENTA, "err: foo") + log(LoggerOutput.StdOut, LoggerLevel.Error, AnsiColor.BLUE, "out: foo") + } + + def name(): String = loggerName + def level(): String = logger.level.toString() + } + + val testClass = new ClassTraitLoggingTest() + + assert(testClass.name() == "io.viash.helpers.LoggerTest$ClassTraitLoggingTest$1") + assert(testClass.level() == "Trace") + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + Console.withOut(outStream) { + Console.withErr(errStream) { + testClass.bar() + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + val expectOut = + s"""out: error + |out: warn + |out: info + |out: debug + |out: trace + |out: success + |out: foo + |""".stripMargin + + val expectErr = + s"""err: error true + |err: warn true + |err: info true + |err: debug true + |err: trace true + |err: success + |err: foo + |""".stripMargin + + assert(stdout == expectOut) + assert(stderr == expectErr) + + // Tack on tests for a variant class + class ClassTraitLoggingTest2 extends ClassTraitLoggingTest + val testClass2 = new ClassTraitLoggingTest2() + + assert(testClass2.name() == "io.viash.helpers.LoggerTest$ClassTraitLoggingTest2$1") + assert(testClass2.level() == "Info") + + val outStream2 = new ByteArrayOutputStream() + val errStream2 = new ByteArrayOutputStream() + Console.withOut(outStream2) { + Console.withErr(errStream2) { + testClass2.bar() + } + } + + val stdout2 = outStream2.toString + val stderr2 = errStream2.toString + + val expectOut2 = + s"""out: error + |out: warn + |out: info + |out: success + |out: foo + |""".stripMargin + + val expectErr2 = + s"""err: error true + |err: warn true + |err: info true + |err: success + |err: foo + |""".stripMargin + + assert(stdout2 == expectOut2) + assert(stderr2 == expectErr2) + } + + // We can't really test the colorize or loglevel options as the singletons would need to be recreated. + // However we can verify that the parsing happened correctly and set the inner logger values correctly. + + test("Check without --colorize option") { + Logger.UseColorOverride.value = Some(false) + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + ) + assert(Logger.UseColorOverride.value == Some(false)) + + Logger.UseColorOverride.value = Some(true) + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + ) + assert(Logger.UseColorOverride.value == Some(true)) + + Logger.UseColorOverride.value = Some(false) + } + + test("Check --colorize true option") { + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + "--colorize", "true" + ) + assert(Logger.UseColorOverride.value == Some(true)) + Logger.UseColorOverride.value = Some(false) + } + + test("Check --colorize false option") { + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + "--colorize", "false" + ) + assert(Logger.UseColorOverride.value == Some(false)) + Logger.UseColorOverride.value = Some(false) + } + + test("Check --colorize auto option") { + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + "--colorize", "auto" + ) + assert(Logger.UseColorOverride.value == None) + Logger.UseColorOverride.value = Some(false) + } + + test("Check --loglevel debug") { + assert(Logger.UseLevelOverride.value == LoggerLevel.Info) + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + "--loglevel", "debug" + ) + assert(Logger.UseLevelOverride.value == LoggerLevel.Debug) + Logger.UseLevelOverride.value = LoggerLevel.Info + } + + test("Check without --loglevel set") { + assert(Logger.UseLevelOverride.value == LoggerLevel.Info) + TestHelper.testMainException[FileNotFoundException]( + "config", "view", "missing.vsh.yaml", + ) + assert(Logger.UseLevelOverride.value == LoggerLevel.Info) + Logger.UseLevelOverride.value = LoggerLevel.Info + } + +} diff --git a/src/test/scala/io/viash/helpers/circe/Convert.scala b/src/test/scala/io/viash/helpers/circe/Convert.scala new file mode 100644 index 000000000..8a00dc324 --- /dev/null +++ b/src/test/scala/io/viash/helpers/circe/Convert.scala @@ -0,0 +1,76 @@ +package io.viash.helpers.circe + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import io.circe._ +import io.viash.exceptions.{ConfigYamlException, ConfigParserException} + +import shapeless.Lazy +import scala.reflect.runtime.universe._ + +import io.circe.Decoder +import io.circe.generic.extras.decoding.ConfiguredDecoder +import io.circe.generic.extras.semiauto.deriveConfiguredDecoder +import io.viash.helpers.Logger + +class ConvertTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + + case class Foo ( + a: String, + b: Int + ) + implicit val decodeFoo: Decoder[Foo] = deriveConfiguredDecoder + + + test("can convert valid yaml") { + val inputText = """ + |a: foo + |b: 5 + """.stripMargin + val expectedOut = Json.fromJsonObject(JsonObject( + "a" -> Json.fromString("foo"), + "b" -> Json.fromInt(5) + )) + + val out = Convert.textToJson(inputText, "foo") + + assert(out == expectedOut) + } + + test("invalid yaml throws an exception") { + val inputText = """ + |a: foo + | b: 5 + """.stripMargin + + assertThrows[ConfigYamlException] { + Convert.textToJson(inputText, "foo") + } + } + + test("valid class json converts to class") { + val inputText = """ + |a: foo + |b: 5 + """.stripMargin + + val yaml = Convert.textToJson(inputText, "foo") + val foo = Convert.jsonToClass[Foo](yaml, "foo") + } + + test("invalid class json throws an exception") + { + val inputText = """ + |a: foo + |c: 5 + """.stripMargin + + val yaml = Convert.textToJson(inputText, "foo") + assertThrows[ConfigParserException] { + Convert.jsonToClass[Foo](yaml, "foo") + } + } + + +} \ No newline at end of file diff --git a/src/test/scala/io/viash/helpers/circe/JMapTest.scala b/src/test/scala/io/viash/helpers/circe/JMapTest.scala index 931ce3e36..d00fb8ec9 100644 --- a/src/test/scala/io/viash/helpers/circe/JMapTest.scala +++ b/src/test/scala/io/viash/helpers/circe/JMapTest.scala @@ -4,8 +4,10 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.{parser => YamlParser} +import io.viash.helpers.Logger class JMapTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) test("checking whether JMap works") { val out = JMap( "foo" -> JMap( diff --git a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala index 8dd875236..60548d687 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseEitherTest.scala @@ -5,8 +5,11 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.Logger class ParseEitherTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + case class A(foo: String) case class B(bar: Int) case class XXX(ab: Either[A, B]) diff --git a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala index 62550e0ae..763a01c3c 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseOneOrMoreTest.scala @@ -7,8 +7,11 @@ import io.circe.yaml.parser import io.viash.helpers.circe._ import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} import io.viash.helpers.data_structures._ +import io.viash.helpers.Logger class ParseOneOrMoreTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + case class XXX(str: OneOrMore[String]) implicit val encodeXXX: Encoder.AsObject[XXX] = deriveConfiguredEncoder implicit val decodeXXX: Decoder[XXX] = deriveConfiguredDecoder diff --git a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala index 6be3aa913..063b6c7cd 100644 --- a/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala +++ b/src/test/scala/io/viash/helpers/circe/ParseStringLikeTest.scala @@ -5,8 +5,11 @@ import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.Logger class ParseStringLikeTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + case class XXX(str: String) implicit val encodeXXX: Encoder.AsObject[XXX] = deriveConfiguredEncoder implicit val decodeXXX: Decoder[XXX] = deriveConfiguredDecoder diff --git a/src/test/scala/io/viash/helpers/circe/RichJsonObjectTest.scala b/src/test/scala/io/viash/helpers/circe/RichJsonObjectTest.scala index 9115d2f8e..8290af347 100644 --- a/src/test/scala/io/viash/helpers/circe/RichJsonObjectTest.scala +++ b/src/test/scala/io/viash/helpers/circe/RichJsonObjectTest.scala @@ -3,8 +3,10 @@ package io.viash.helpers.circe import io.circe.Json import io.circe.JsonObject import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger class RichJsonObjectTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) test("RichJsonObject.map should apply function to all key-value pairs") { val inputJson = JsonObject( diff --git a/src/test/scala/io/viash/helpers/circe/RichJsonTest.scala b/src/test/scala/io/viash/helpers/circe/RichJsonTest.scala index ba0be03cf..5aefc796d 100644 --- a/src/test/scala/io/viash/helpers/circe/RichJsonTest.scala +++ b/src/test/scala/io/viash/helpers/circe/RichJsonTest.scala @@ -4,10 +4,11 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.parser -import io.viash.helpers.IO +import io.viash.helpers.{IO, Logger} import java.nio.file.Files class RichJsonTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) private val temporaryFolder = IO.makeTemp("richjson") test("checking whether withDefault works") { diff --git a/src/test/scala/io/viash/helpers/circe/ValidationTest.scala b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala new file mode 100644 index 000000000..c8c1ea792 --- /dev/null +++ b/src/test/scala/io/viash/helpers/circe/ValidationTest.scala @@ -0,0 +1,145 @@ +package io.viash.helpers.circe + +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import io.circe._ +import io.circe.yaml.parser +import io.viash.helpers.circe._ +import io.circe.generic.extras.semiauto.{deriveConfiguredDecoder, deriveConfiguredEncoder} +import io.viash.helpers.data_structures._ +import io.viash.helpers.Logger +import java.io.ByteArrayOutputStream +import io.viash.schemas._ +import io.viash.exceptions.ConfigParserValidationException + +class ValidationTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) + + @deprecated("testing class deprecation.", "0.1.2", "987.654.321") + case class TestClassDeprecation( + foo: String + ) + @removed("testing class removal.", "0.0.1", "0.1.2") + case class TestClassRemoval( + foo: String + ) + case class TestClassWithFieldDeprecation( + @deprecated("testing deprecation of foo.", "0.1.2", "987.654.321") + foo: String + ) + case class TestClassWithFieldRemoval( + @removed("testing removal of foo.", "0.0.1", "0.1.2") + foo: String + ) + case class TestClassValidation( + bar: String + ) + implicit val decodeDeprecation: Decoder[TestClassDeprecation] = DeriveConfiguredDecoderWithDeprecationCheck.deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeRemoval: Decoder[TestClassRemoval] = DeriveConfiguredDecoderWithDeprecationCheck.deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeFieldDeprecation: Decoder[TestClassWithFieldDeprecation] = DeriveConfiguredDecoderWithDeprecationCheck.deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeFieldRemoval: Decoder[TestClassWithFieldRemoval] = DeriveConfiguredDecoderWithDeprecationCheck.deriveConfiguredDecoderWithDeprecationCheck + implicit val decodeValidation: Decoder[TestClassValidation] = DeriveConfiguredDecoderWithValidationCheck.deriveConfiguredDecoderWithValidationCheck + + test("parsing of a deprecated class") { + val json = parser.parse("foo: bar").getOrElse(Json.Null) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + val parsed = Console.withOut(outStream) { + Console.withErr(errStream) { + json.as[TestClassDeprecation].toOption.get + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr.contains("Warning: TestClassDeprecation is deprecated: testing class deprecation. Deprecated since 0.1.2, planned removal 987.654.321.")) + + assert(parsed == TestClassDeprecation(foo = "bar")) + } + + test("parsing of a removed class") { + val json = parser.parse("foo: bar").getOrElse(Json.Null) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + val parsed = Console.withOut(outStream) { + Console.withErr(errStream) { + json.as[TestClassRemoval].toOption.get + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr.contains("Error: TestClassRemoval was removed: testing class removal. Initially deprecated 0.0.1, removed 0.1.2.")) + + assert(parsed == TestClassRemoval(foo = "bar")) + } + + + test("parsing of a class with a deprecated field") { + val json = parser.parse("foo: bar").getOrElse(Json.Null) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + val parsed = Console.withOut(outStream) { + Console.withErr(errStream) { + json.as[TestClassWithFieldDeprecation].toOption.get + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr.contains("Warning: ..foo is deprecated: testing deprecation of foo. Deprecated since 0.1.2, planned removal 987.654.321.")) + + assert(parsed == TestClassWithFieldDeprecation(foo = "bar")) + } + + test("parsing of a class with a removed field") { + val json = parser.parse("foo: bar").getOrElse(Json.Null) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + val parsed = Console.withOut(outStream) { + Console.withErr(errStream) { + json.as[TestClassWithFieldRemoval].toOption.get + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr.contains("Error: ..foo was removed: testing removal of foo. Initially deprecated 0.0.1, removed 0.1.2.")) + + assert(parsed == TestClassWithFieldRemoval(foo = "bar")) + } + + test("parsing of a structure that does not match class definition") { + val json = parser.parse("foo: bar").getOrElse(Json.Null) + + val outStream = new ByteArrayOutputStream() + val errStream = new ByteArrayOutputStream() + val exception = intercept[ConfigParserValidationException] { + Console.withOut(outStream) { + Console.withErr(errStream) { + json.as[TestClassValidation].toOption.get + } + } + } + + val stdout = outStream.toString + val stderr = errStream.toString + + assert(stdout.isEmpty()) + assert(stderr.isEmpty()) + assert(exception.toString().contains("Invalid data fields for TestClassValidation.")) + } + +} \ No newline at end of file diff --git a/src/test/scala/io/viash/helpers/circe/YamlTest.scala b/src/test/scala/io/viash/helpers/circe/YamlTest.scala index 0023f18c9..66146435a 100644 --- a/src/test/scala/io/viash/helpers/circe/YamlTest.scala +++ b/src/test/scala/io/viash/helpers/circe/YamlTest.scala @@ -4,8 +4,12 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.circe._ import io.circe.yaml.{parser => YamlParser} +import io.viash.helpers.{Yaml => YamlHelper} +import io.viash.helpers.Logger class YamlTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + test("checking whether Yaml works") { val out = Yaml(""" |foo: @@ -25,4 +29,16 @@ class YamlTest extends AnyFunSuite with BeforeAndAfterAll { assert(out == expectedOut) } + + test(".inf values can be read by circe after \"fixing\" them") { + // Expected behaviour is that circe still doesn't accept yaml +.inf, -.inf, or .nan number values. + assertThrows[ParsingFailure] { + val failure = Yaml("""val: +.inf""") + } + + val out1 = Yaml("""val: "+.inf"""") + val out2 = Yaml(YamlHelper.replaceInfinities("""val: +.inf""")) + + assert(out1 == out2) + } } \ No newline at end of file diff --git a/src/test/scala/io/viash/helpers/data_structures/OneOrMoreTest.scala b/src/test/scala/io/viash/helpers/data_structures/OneOrMoreTest.scala index 1cb088de5..a464f7f40 100644 --- a/src/test/scala/io/viash/helpers/data_structures/OneOrMoreTest.scala +++ b/src/test/scala/io/viash/helpers/data_structures/OneOrMoreTest.scala @@ -2,8 +2,11 @@ package io.viash.helpers.data_structures import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite +import io.viash.helpers.Logger class OneOrMoreTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) + test("works with one element") { val oom = OneOrMore("foo") assert(oom.toList == List("foo")) diff --git a/src/test/scala/io/viash/platforms/NextFlowUtilsTest.scala b/src/test/scala/io/viash/platforms/NextFlowUtilsTest.scala deleted file mode 100644 index 62452d08e..000000000 --- a/src/test/scala/io/viash/platforms/NextFlowUtilsTest.scala +++ /dev/null @@ -1,47 +0,0 @@ -package io.viash.platforms - -import io.viash.DockerTest -import io.viash.platforms.NextFlowUtils._ -import org.scalatest.funsuite.AnyFunSuite - -import scala.util.Try - -class NextFlowUtilsTest extends AnyFunSuite { - - val simpleTuple1:ConfigTuple = ("key", "value") - val simpleTuple2:ConfigTuple = ("key", true) - val simpleTuple3:ConfigTuple = ("key", 2) - - val listTuple:ConfigTuple = ("key", List("value1", "value2")) - - val nestedTuple:ConfigTuple = ("key", NestedValue(List(simpleTuple1, simpleTuple2, simpleTuple3, listTuple))) - - // convert testbash - test("NextFlowPlatform can deal with simple and nested Tuples") { - assert(simpleTuple1.isInstanceOf[ConfigTuple]) - assert(nestedTuple.isInstanceOf[ConfigTuple]) - } - - test("Tuples can be implicitly converted to ConfigTuples for plain values") { - assert(Try(("key", "values").toConfig()).toOption.isDefined) - } - - test("ConfigTuples can be exported to config String") { - assert(simpleTuple1.toConfig("") === """key = "value"""") - assert(simpleTuple2.toConfig("") === """key = true""") - assert(simpleTuple3.toConfig("") === """key = 2""") - } - - test("Nested ConfigTuples can be exported to String properly as well", DockerTest) { - val configString = nestedTuple.toConfig(" ") - val expectedString = - """ key { - | key = "value" - | key = true - | key = 2 - | key = [ "value1", "value2" ] - | }""" - .stripMargin - assert(configString === expectedString) - } -} diff --git a/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala b/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala index c332d8b69..bb2293808 100644 --- a/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala @@ -10,11 +10,12 @@ import java.io.UncheckedIOException import scala.io.Source -import io.viash.helpers.IO -import io.viash.{DockerTest, NextFlowTest, TestHelper} +import io.viash.helpers.{IO, Logger} +import io.viash.{DockerTest, NextflowTest, TestHelper} import io.viash.NextflowTestHelper class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // temporary folder to work in private val temporaryFolder = IO.makeTemp("viash_tester_nextflowvdsl3") private val tempFolFile = temporaryFolder.toFile @@ -109,7 +110,7 @@ class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll { for (resource <- List("src", "workflows", "resources")) TestHelper.copyFolder(Paths.get(rootPath, resource).toString, Paths.get(tempFolStr, resource).toString) - test("Build pipeline components", DockerTest, NextFlowTest) { + test("Build pipeline components", DockerTest, NextflowTest) { // build the nextflow containers val (_, _, _) = TestHelper.testMainWithStdErr( "ns", "build", @@ -119,94 +120,50 @@ class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll { ) } - test("Run pipeline", DockerTest, NextFlowTest) { - + test("Run pipeline", DockerTest, NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( mainScript = "workflows/pipeline1/main.nf", entry = Some("base"), - args = List( - "--input", "resources/*", - "--publish_dir", "output", - ), + args = List("--publish_dir", "output"), cwd = tempFolFile ) assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") outputFileMatchChecker(stdOut, "DEBUG6", "^11 .*$") - } - - test("Run pipeline with components using map functionality", DockerTest, NextFlowTest) { - - val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( - mainScript = "workflows/pipeline1/main.nf", - entry = Some("map_variant"), - args = List( - "--input", "resources/*", - "--publish_dir", "output", - ), - cwd = tempFolFile - ) - - assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - outputFileMatchChecker(stdOut, "DEBUG4", "^11 .*$") - } - - test("Run pipeline with components using mapData functionality", DockerTest, NextFlowTest) { - val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( - mainScript = "workflows/pipeline1/main.nf", - entry = Some("mapData_variant"), - args = List( - "--input", "resources/*", - "--publish_dir", "output", - ), - cwd = tempFolFile - ) + // check whether step3's debug printing was triggered + outputFileMatchChecker(stdOut, "process 'step3[^']*' output tuple", "^11 .*$") - assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - outputFileMatchChecker(stdOut, "DEBUG4", "^11 .*$") + // check whether step2's debug printing was not triggered + val lines2 = stdOut.split("\n").find(_.contains("process 'step2' output tuple")) + assert(!lines2.isDefined) } - test("Run pipeline with debug = false", DockerTest, NextFlowTest) { + test("Test map/mapData/id arguments", DockerTest, NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( mainScript = "workflows/pipeline1/main.nf", - entry = Some("debug_variant"), - args = List( - "--input", "resources/*", - "--publish_dir", "output", - "--displayDebug", "false", - ), + entry = Some("test_map_mapdata_mapid_arguments"), + args = List("--publish_dir", "output"), cwd = tempFolFile ) assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - outputFileMatchChecker(stdOut, "DEBUG4", "^11 .*$") - - val lines2 = stdOut.split("\n").find(_.contains("process 'step3' output tuple")) - assert(!lines2.isDefined) - } - test("Run pipeline with debug = true", DockerTest, NextFlowTest) { + test("Test fromState/toState arguments", DockerTest, NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( mainScript = "workflows/pipeline1/main.nf", - entry = Some("debug_variant"), - args = List( - "--input", "resources/*", - "--publish_dir", "output", - "--displayDebug", "true", - ), + entry = Some("test_fromstate_tostate_arguments"), + args = List("--publish_dir", "output"), cwd = tempFolFile ) assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - outputFileMatchChecker(stdOut, "DEBUG4", "^11 .*$") - outputFileMatchChecker(stdOut, "process 'step3[^']*' output tuple", "^11 .*$") } - test("Check whether --help is same as Viash's --help", NextFlowTest) { + test("Check whether --help is same as Viash's --help", NextflowTest) { // except that WorkflowHelper.nf will not print alternatives, and // will always prefix argument names with -- (so --foo, not -f or foo). diff --git a/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala b/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala index aaa57ea3d..9ac0d0490 100644 --- a/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/Vdsl3StandaloneTest.scala @@ -9,12 +9,13 @@ import java.io.File import java.nio.file.{Files, Path, Paths} import scala.io.Source -import io.viash.helpers.IO -import io.viash.{DockerTest, NextFlowTest, TestHelper} +import io.viash.helpers.{IO, Logger} +import io.viash.{DockerTest, NextflowTest, TestHelper} import io.viash.NextflowTestHelper import java.nio.charset.StandardCharsets class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // temporary folder to work in private val temporaryFolder = IO.makeTemp("viash_tester_nextflowvdsl3") private val tempFolFile = temporaryFolder.toFile @@ -34,7 +35,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { for (resource <- List("src", "workflows", "resources")) TestHelper.copyFolder(Paths.get(rootPath, resource).toString, Paths.get(tempFolStr, resource).toString) - test("Build pipeline components", DockerTest, NextFlowTest) { + test("Build pipeline components", DockerTest, NextflowTest) { // build the nextflow containers val (_, _, _) = TestHelper.testMainWithStdErr( "ns", "build", @@ -44,7 +45,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { ) } - test("Simple run", NextFlowTest) { + test("Simple run", NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( mainScript = "target/nextflow/step2/main.nf", args = List( @@ -66,7 +67,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { } } - test("With id containing spaces and slashes", NextFlowTest) { + test("With id containing spaces and slashes", NextflowTest) { val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( mainScript = "target/nextflow/step2/main.nf", args = List( @@ -89,7 +90,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { } } - test("With yamlblob param_list", NextFlowTest) { + test("With yamlblob param_list", NextflowTest) { val paramListStr = "[{input1: resources/lines3.txt, input2: resources/lines5.txt}]" val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( mainScript = "target/nextflow/step2/main.nf", @@ -111,7 +112,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { } } - test("With yaml param_list", NextFlowTest) { + test("With yaml param_list", NextflowTest) { val paramListPath = temporaryFolder.resolve("resources/param_list.yaml") val paramListStr = "- input1: lines3.txt\n input2: lines5.txt" @@ -138,7 +139,7 @@ class Vdsl3StandaloneTest extends AnyFunSuite with BeforeAndAfterAll { } } - test("With optional inputs", NextFlowTest) { + test("With optional inputs", NextflowTest) { Files.copy(Paths.get(resourcesPath, "lines5.txt"), Paths.get(resourcesPath, "lines5-bis.txt")) diff --git a/src/test/scala/io/viash/platforms/nextflow/WorkflowHelperTest.scala b/src/test/scala/io/viash/platforms/nextflow/WorkflowHelperTest.scala index 453582678..ce1a1751f 100644 --- a/src/test/scala/io/viash/platforms/nextflow/WorkflowHelperTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/WorkflowHelperTest.scala @@ -1,7 +1,7 @@ package io.viash.platforms.nextflow -import io.viash.helpers.IO -import io.viash.{DockerTest, NextFlowTest, TestHelper} +import io.viash.helpers.{IO, Logger} +import io.viash.{DockerTest, NextflowTest, TestHelper} import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite @@ -13,6 +13,7 @@ import java.io.IOException import java.io.UncheckedIOException class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) // temporary folder to work in private val temporaryFolder = IO.makeTemp("viash_tester_nextflowvdsl3") private val tempFolStr = temporaryFolder.toString @@ -139,7 +140,7 @@ class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll { for (resource <- List("src", "workflows", "resources")) TestHelper.copyFolder(Paths.get(rootPath, resource).toString, Paths.get(tempFolStr, resource).toString) - test("Build pipeline components", DockerTest, NextFlowTest) { + test("Build pipeline components", DockerTest, NextflowTest) { // build the nextflow containers val (_, _, _) = TestHelper.testMainWithStdErr( "ns", "build", @@ -170,7 +171,7 @@ class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll { NotAvailCheck("multiple") ) - test("Run config pipeline", NextFlowTest) { + test("Run config pipeline", NextflowTest) { val (exitCode, stdOut, stdErr) = runNextflowProcess( mainScript = "workflows/pipeline3/main.nf", @@ -193,7 +194,7 @@ class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll { checkDebugArgs("foo", debugPrints, expectedFoo) } - test("Run config pipeline with yamlblob", NextFlowTest) { + test("Run config pipeline with yamlblob", NextflowTest) { val fooArgs = "{id: foo, input: resources/lines3.txt, whole_number: 3, optional_with_default: foo, multiple: [a, b, c]}" val barArgs = "{id: bar, input: resources/lines5.txt, real_number: 0.5, optional: bar, reality: true}" @@ -218,7 +219,7 @@ class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll { assert(debugPrints.find(_._1 == "foo").get._2("input").endsWith(resourcesPath+"/lines3.txt")) } - test("Run config pipeline with yaml file", NextFlowTest) { + test("Run config pipeline with yaml file", NextflowTest) { val param_list_file = Paths.get(resourcesPath, "pipeline3.yaml").toFile.toString val (exitCode, stdOut, stdErr) = runNextflowProcess( mainScript = "workflows/pipeline3/main.nf", @@ -241,7 +242,7 @@ class WorkflowHelperTest extends AnyFunSuite with BeforeAndAfterAll { assert(debugPrints.find(_._1 == "foo").get._2("input").endsWith(resourcesPath+"/lines3.txt")) } -test("Run config pipeline with yaml file passed as a relative path", NextFlowTest) { +test("Run config pipeline with yaml file passed as a relative path", NextflowTest) { val (exitCode, stdOut, stdErr) = runNextflowProcess( mainScript = "../workflows/pipeline3/main.nf", entry = Some("base"), @@ -264,7 +265,7 @@ test("Run config pipeline with yaml file passed as a relative path", NextFlowTes } - test("Run config pipeline with json file", NextFlowTest) { + test("Run config pipeline with json file", NextflowTest) { val param_list_file = Paths.get(resourcesPath, "pipeline3.json").toFile.toString val (exitCode, stdOut, stdErr) = runNextflowProcess( mainScript = "workflows/pipeline3/main.nf", @@ -287,7 +288,7 @@ test("Run config pipeline with yaml file passed as a relative path", NextFlowTes assert(debugPrints.find(_._1 == "foo").get._2("input").endsWith(resourcesPath+"/lines3.txt")) } - test("Run config pipeline with csv file", NextFlowTest) { + test("Run config pipeline with csv file", NextflowTest) { val param_list_file = Paths.get(resourcesPath, "pipeline3.csv").toFile.toString val (exitCode, stdOut, stdErr) = runNextflowProcess( mainScript = "workflows/pipeline3/main.nf", @@ -310,7 +311,7 @@ test("Run config pipeline with yaml file passed as a relative path", NextFlowTes assert(debugPrints.find(_._1 == "foo").get._2("input").endsWith(resourcesPath+"/lines3.txt")) } - test("Run config pipeline asis, default nextflow implementation", NextFlowTest) { + test("Run config pipeline asis, default nextflow implementation", NextflowTest) { val param_list_file = Paths.get(resourcesPath, "pipeline3.asis.yaml").toFile.toString val (exitCode, stdOut, stdErr) = runNextflowProcess( mainScript = "workflows/pipeline3/main.nf", diff --git a/src/test/scala/io/viash/project/ProjectTest.scala b/src/test/scala/io/viash/project/ProjectTest.scala index c60b966ac..074e628e7 100644 --- a/src/test/scala/io/viash/project/ProjectTest.scala +++ b/src/test/scala/io/viash/project/ProjectTest.scala @@ -4,8 +4,10 @@ import io.circe.Json import org.scalatest.funsuite.AnyFunSuite import io.circe.syntax._ import java.nio.file.Paths +import io.viash.helpers.Logger class ProjectTest extends AnyFunSuite { + Logger.UseColorOverride.value = Some(false) private val rootPath = Paths.get(getClass.getResource("/").getPath) private val testBashPath = rootPath.resolve("testbash") private val testNsPath = rootPath.resolve("testns") diff --git a/src/test/scala/io/viash/schema/SchemaTest.scala b/src/test/scala/io/viash/schema/SchemaTest.scala index 7097cb5d5..fbe5547c4 100644 --- a/src/test/scala/io/viash/schema/SchemaTest.scala +++ b/src/test/scala/io/viash/schema/SchemaTest.scala @@ -4,15 +4,22 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import io.viash.schemas.CollectedSchemas import scala.sys.process.Process +import io.viash.helpers.Logger class SchemaTest extends AnyFunSuite with BeforeAndAfterAll { + Logger.UseColorOverride.value = Some(false) test("All schema class val members should be annotated") { val nonAnnotated = CollectedSchemas.getAllNonAnnotated - nonAnnotated.foreach { - case (key, key2, member) => Console.err.println(s"$key - $key2 - $member") + + assert(nonAnnotated.contains("CollectedSchemas")) + assert(nonAnnotated("CollectedSchemas") == "__this__") + + nonAnnotated.removed("CollectedSchemas").foreach { + case (key, member) => Console.err.println(s"$key - $member") } - assert(nonAnnotated.size == 0) + + assert(nonAnnotated.size == 1) } test("Check formatting of deprecation annotations") { diff --git a/src/viash/viash_build/config.vsh.yaml b/src/viash/viash_build/config.vsh.yaml deleted file mode 100644 index efa2b0a9e..000000000 --- a/src/viash/viash_build/config.vsh.yaml +++ /dev/null @@ -1,97 +0,0 @@ -functionality: - name: viash_build - namespace: viash - description: | - Build a project, usually in the context of a pipeline. - arguments: - - name: "--src" - alternatives: [ "-s" ] - type: file - description: Directory for sources if different from src/ - default: src - - name: "--mode" - alternatives: [ "-m" ] - type: string - description: "The mode to run in. Possible values are: 'development', 'integration', 'release'." - default: development - - name: "--platform" - alternatives: [ "-p" ] - type: string - description: "Which platforms to process." - example: "docker|nextflow" - - name: "--query" - alternatives: [ "-q" ] - type: string - description: "Filter which components get selected by component and namespace name. Can be a regex." - example: "^mynamespace/component1$" - - name: "--query_namespace" - alternatives: [ "-n" ] - type: string - description: "Filter which namespaces get selected by namespace name. Can be a regex." - example: "^mynamespace$" - - name: "--query_name" - type: string - description: "Filter which components get selected by component name. Can be a regex." - example: "^component1$" - - name: "--tag" - alternatives: [ "-t" ] - type: string - description: The tag/version to be used. - example: "0.1.0" - - name: "--registry" - alternatives: [ "-r" ] - example: ghcr.io - type: string - description: Which Docker registry to use in the Docker image name. - - name: "--organization" - alternatives: [ "-o", "--organisation" ] - example: myorganisation - type: string - description: Which organisation name to use in the Docker image name. - - name: "--target_image_source" - alternatives: [ "-tis" ] - type: string - description: Which image source to specify in the component builds. - example: https://github.com/myorganisation/myrepository - - name: "--namespace_separator" - example: "_" - type: string - description: The separator to use between the component name and namespace as the image name of a Docker container. - - name: "--nextflow_variant" - type: string - description: "[Deprecated] Which nextflow variant to use." - - name: "--max_threads" - type: integer - description: The maximum number of threads viash will use when `--parallel` during parallel tasks. - example: 8 - - name: "--config_mod" - alternatives: [ "-c" ] - type: string - multiple: true - multiple_sep: ";" - description: "Modify a viash config at runtime using a custom DSL." - - name: "--no_cache" - alternatives: [ "-nc", "--no-cache" ] - type: boolean_true - description: Don't cache the docker build in development mode. - - name: "--log" - alternatives: [ "-l" ] - type: file - description: Log file - example: .viash_build_log.txt - direction: output - - name: "--viash" - type: file - description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. - - name: "--verbose" - type: boolean_true - description: "Increase verbosity." - resources: - - type: bash_script - path: script.sh - test_resources: - - type: bash_script - path: run_test.sh - - path: ../../test/resources/testns/src -platforms: -- type: native diff --git a/src/viash/viash_build/run_test.sh b/src/viash/viash_build/run_test.sh deleted file mode 100644 index 6f9d933ea..000000000 --- a/src/viash/viash_build/run_test.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -defaults_output="defaults_test_output.txt" -expected_target_dir="target/native/testns" - -alt_output=alt_test_output.txt -alt_src="alt_src" -log=build_log.txt -target_ns_add="target/native/testns/ns_add/ns_add" -target_ns_add_yaml="target/native/testns/ns_add/.config.vsh.yaml" - -# 1. Run component with default arguments -# Run component -$meta_executable \ - --verbose \ - >$defaults_output - -# Check if defaults output exists -[[ ! -f $defaults_output ]] && echo "Default: Test output file could not be found!" && exit 1 - -# Check if default arguments are as expected -grep -q "viash ns build --src src --parallel --config_mod .functionality.version := 'dev' --setup cachedbuild" $defaults_output - -# Check if target dir hierarchy exists -[[ ! -d $expected_target_dir ]] && echo "Default: target directory hierarchy could not be found!" && exit 1 - -# Remove target dir -rm -r target - -# 2. Run component with custom arguments -# Copy src dir -cp -r src $alt_src - -# Run component -$meta_executable \ - --verbose \ - --src $alt_src \ - --platform native \ - --mode release \ - --tag rc-1 \ - --query_namespace 'testns' \ - --query_name 'ns_a' \ - --registry 'my_registry' \ - --organization 'my_organization' \ - --target_image_source 'https://github.com/viash-io/viash' \ - --namespace_separator '*~*' \ - --max_threads 8 \ - --config_mod '.functionality.version := "5.0"' \ - --log $log - ->$alt_output - -# Check if alt output exists -[[ ! -f $alt_output ]] && echo "Alt: Test output file could not be found!" && exit 1 - -# Only ns_add should be included because of the query mame -[[ -d "$expected_target_dir/ns_divide" ]] && echo "The ns_divide component shouldn't have been built!" && exit 1 - -# The build log should contain an error as ns_error can't be built -grep -q "Reading file 'alt_src/ns_error/config.vsh.yaml' failed" $log - -# Check if target dir hierarchy exists -[[ ! -d $expected_target_dir ]] && echo "Alt: target directory hierarchy could not be found!" && exit 1 - -# Check if ns_add exists in target dir -[[ ! -f $target_ns_add ]] && echo "Alt: ns_add couldn't be found in target directory!" && exit 1 - -# Check if the version of ns_add is changed to 5.0 -grep -q "ns_add 5.0" $target_ns_add - -# Check if the namespace of ns_add is testns -grep -q 'namespace: "testns"' $target_ns_add_yaml - -echo ">>> Test finished successfully" diff --git a/src/viash/viash_build/script.sh b/src/viash/viash_build/script.sh deleted file mode 100644 index e22896892..000000000 --- a/src/viash/viash_build/script.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash - -# start creating command -command_builder=( - ns build - --src "$par_src" - --parallel -) - -# check par mode -if [ "$par_mode" == "development" ]; then - echo "In development mode with 'dev'." -elif [ "$par_mode" == "integration" ]; then - echo "In integration mode with tag '$par_tag'." -elif [ "$par_mode" == "release" ]; then - echo "In RELEASE mode with tag '$par_tag'." -else - echo "Error: Not a valid mode argument '$par_mode'." - exit 1 -fi - -# check tag -if [ "$par_mode" == "development" ]; then - if [ ! -z "$par_tag" ]; then - echo "Warning: '--tag' is ignored when '--mode=$par_mode'." - fi - par_tag="dev" -fi -if [ -z "$par_tag" ]; then - echo "Error: --tag is a requirement argument when '--mode=$par_mode'." - exit 1 -fi - -command_builder+=( - --config_mod ".functionality.version := '$par_tag'" -) - -# derive setup strategy -if [ "$par_mode" == "development" ]; then - if [ "$par_no_cache" == "true" ]; then - setup_strat="build" - else - setup_strat="cachedbuild" - fi -elif [ "$par_mode" == "integration" ]; then - echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." - setup_strat="ifneedbepullelsecachedbuild" -elif [ "$par_mode" == "release" ]; then - echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." - setup_strat="build" -fi - -command_builder+=( - --setup "$setup_strat" -) - -# check registry and organization -if [ "$par_mode" == "development" ]; then - if [ ! -z "$par_registry" ]; then - [[ "$par_verbose" == "true" ]] && echo "Note: --par_registry is ignored when '--mode=development'." - unset par_registry - fi - - if [ ! -z "$par_organization" ]; then - [[ "$par_verbose" == "true" ]] && echo "Note: --par_organization is ignored when '--mode=development'." - unset par_organization - fi -fi - -################ COMMON PARAMS ################ - -# check viash arg -# if not specified, default par_viash to look for 'viash' on the PATH -if [ -z "$par_viash" ]; then - par_viash="viash" -fi - -# if specified, use par_max_threads as a java argument -if [ ! -z "$par_max_threads" ]; then - export JAVA_ARGS="$JAVA_ARGS -Dscala.concurrent.context.maxThreads=$par_max_threads" -fi - -# process queries -if [ ! -z "$par_query" ]; then - command_builder+=("--query" "$par_query") -fi -if [ ! -z "$par_query_name" ]; then - command_builder+=("--query_name" "$par_query_name") -fi -if [ ! -z "$par_query_namespace" ]; then - command_builder+=("--query_namespace" "$par_query_namespace") -fi - -# process config mods -if [ ! -z "$par_config_mod" ]; then - IFS=";" - for var in $par_config_mod; do - unset IFS - command_builder+=("--config_mod" "$var") - done -fi - -if [ ! -z "$par_registry" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_registry := '$par_registry'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].registry := '$par_registry'" - ) -fi - -if [ ! -z "$par_organization" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_organization := '$par_organization'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].organization := '$par_organization'" - ) -fi - -if [ ! -z "$par_namespace_separator" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].namespace_separator := '$par_namespace_separator'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].namespace_separator := '$par_namespace_separator'" - ) -fi - -if [ ! -z "$par_target_image_source" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_image_source := '$par_target_image_source'" - ) -fi - -if [ ! -z "$par_platform" ]; then - command_builder+=(--platform "$par_platform") -fi - -################ RUN COMMAND ################ -[[ "$par_verbose" == "true" ]] && echo "+ $par_viash" "${command_builder[@]}" - -if [ -z "$par_log" ]; then - "$par_viash" "${command_builder[@]}" -else - [ ! -f "$par_log" ] || rm "$par_log" - "$par_viash" "${command_builder[@]}" > >(tee -a "$par_log") 2> >(tee -a "$par_log") -fi diff --git a/src/viash/viash_push/config.vsh.yaml b/src/viash/viash_push/config.vsh.yaml deleted file mode 100644 index fab7dd79b..000000000 --- a/src/viash/viash_push/config.vsh.yaml +++ /dev/null @@ -1,84 +0,0 @@ -functionality: - name: viash_push - namespace: viash - description: | - Push a project, usually in the context of a pipeline. - arguments: - - name: "--src" - alternatives: [ "-s" ] - type: file - description: Directory for sources if different from src/ - default: src - - name: "--mode" - alternatives: [ "-m" ] - type: string - description: "The mode to run in. Possible values are: 'development', 'integration', 'release'." - default: development - - name: "--query" - alternatives: [ "-q" ] - type: string - description: "Filter which components get selected by component and namespace name. Can be a regex." - example: "^mynamespace/component1$" - - name: "--query_namespace" - alternatives: [ "-n" ] - type: string - description: "Filter which namespaces get selected by namespace name. Can be a regex." - example: "^mynamespace$" - - name: "--query_name" - type: string - description: "Filter which components get selected by component name. Can be a regex." - example: "^component1$" - - name: "--tag" - alternatives: [ "-t" ] - type: string - description: The tag/version to be used. - example: "0.1.0" - - name: "--registry" - alternatives: [ "-r" ] - example: ghcr.io - type: string - description: Which Docker registry to use in the Docker image name. - - name: "--organization" - alternatives: [ "-o", "--organisation" ] - example: myorganisation - type: string - description: Which organisation name to use in the Docker image name. - - name: "--target_image_source" - alternatives: [ "-tis" ] - type: string - description: Which image source to specify in the component builds. - example: https://github.com/myorganisation/myrepository - - name: "--namespace_separator" - example: "_" - type: string - description: The separator to use between the component name and namespace as the image name of a Docker container. - - name: "--max_threads" - type: integer - description: The maximum number of threads viash will use when `--parallel` during parallel tasks. - example: 8 - - name: "--force" - type: boolean_true - description: Overwrite existing container. - - name: "--config_mod" - alternatives: [ "-c" ] - type: string - multiple: true - multiple_sep: ";" - description: "Modify a viash config at runtime using a custom DSL." - - name: "--log" - alternatives: [ "-l" ] - type: file - description: Log file - example: .viash_push_log.txt - direction: output - - name: "--viash" - type: file - description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. - - name: "--verbose" - type: boolean_true - description: "Increase verbosity." - resources: - - type: bash_script - path: script.sh -platforms: -- type: native diff --git a/src/viash/viash_push/script.sh b/src/viash/viash_push/script.sh deleted file mode 100644 index 2f5280d94..000000000 --- a/src/viash/viash_push/script.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -if [ "$par_force" == "true" ]; then - echo "Force push... handle with care..." - sleep 2 -fi - -# start creating command -command_builder=( - ns build - --src "$par_src" - --platform docker - --parallel -) - -# check par mode -if [ "$par_mode" == "development" ]; then - echo "No container push can and should be performed in this mode." - exit 1 -elif [ "$par_mode" == "integration" ]; then - echo "No container push can and should be performed in this mode." - exit 1 - # if [ ! -z "$par_tag" ]; then - # echo "Warning: '--tag' is ignored when '--mode=$par_mode'." - # fi - # par_tag="dev" - # echo "In integration mode with tag '$par_tag'." -elif [ "$par_mode" == "release" ]; then - echo "In RELEASE mode with tag '$par_tag'." -else - echo "Error: Not a valid mode argument '$par_mode'." - exit 1 -fi - -if [ -z "$par_tag" ]; then - echo "Error: --tag is a requirement argument when '--mode=$par_mode'." - exit 1 -fi - -command_builder+=( - --config_mod ".functionality.version := '$par_tag'" -) - -# derive setup strategy -if [ "$par_force" == "true" ]; then - setup_strat="push" -else - setup_strat="pushifnotpresent" -fi - -command_builder+=( - --setup "$setup_strat" -) - - -################ COMMON PARAMS ################ - -# check viash arg -# if not specified, default par_viash to look for 'viash' on the PATH -if [ -z "$par_viash" ]; then - par_viash="viash" -fi - -# if specified, use par_max_threads as a java argument -if [ ! -z "$par_max_threads" ]; then - export JAVA_ARGS="$JAVA_ARGS -Dscala.concurrent.context.maxThreads=$par_max_threads" -fi - -# process queries -if [ ! -z "$par_query" ]; then - command_builder+=("--query" "$par_query") -fi -if [ ! -z "$par_query_name" ]; then - command_builder+=("--query_name" "$par_query_name") -fi -if [ ! -z "$par_query_namespace" ]; then - command_builder+=("--query_namespace" "$par_query_namespace") -fi - -# process config mods -if [ ! -z "$par_config_mod" ]; then - IFS=";" - for var in $par_config_mod; do - unset IFS - command_builder+=("--config_mod" "$var") - done -fi - -if [ ! -z "$par_registry" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_registry := '$par_registry'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].registry := '$par_registry'" - ) -fi - -if [ ! -z "$par_organization" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_organization := '$par_organization'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].organization := '$par_organization'" - ) -fi - -if [ ! -z "$par_namespace_separator" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].namespace_separator := '$par_namespace_separator'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].namespace_separator := '$par_namespace_separator'" - ) -fi - -if [ ! -z "$par_target_image_source" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_image_source := '$par_target_image_source'" - ) -fi - - -################ RUN COMMAND ################ -[[ "$par_verbose" == "true" ]] && echo "+ $par_viash" "${command_builder[@]}" - -if [ -z "$par_log" ]; then - "$par_viash" "${command_builder[@]}" -else - [ ! -f "$par_log" ] || rm "$par_log" - "$par_viash" "${command_builder[@]}" > >(tee -a "$par_log") 2> >(tee -a "$par_log") -fi diff --git a/src/viash/viash_test/config.vsh.yaml b/src/viash/viash_test/config.vsh.yaml deleted file mode 100644 index a5ed9f672..000000000 --- a/src/viash/viash_test/config.vsh.yaml +++ /dev/null @@ -1,106 +0,0 @@ -functionality: - name: viash_test - namespace: viash - description: | - Test a project, usually in the context of a pipeline. - arguments: - - name: "--src" - alternatives: [ "-s" ] - type: file - description: Directory for sources if different from src/ - default: src - - name: "--mode" - alternatives: [ "-m" ] - type: string - description: "The mode to run in. Possible values are: 'development', 'integration', 'release'." - default: development - - name: "--platform" - alternatives: [ "-p" ] - type: string - description: "Which platforms to process." - default: "docker" # to do: should be set to 'example' - - name: "--query" - alternatives: [ "-q" ] - type: string - description: "Filter which components get selected by component and namespace name. Can be a regex." - example: "^mynamespace/component1$" - - name: "--query_namespace" - alternatives: [ "-n" ] - type: string - description: "Filter which namespaces get selected by namespace name. Can be a regex." - example: "^mynamespace$" - - name: "--query_name" - type: string - description: "Filter which components get selected by component name. Can be a regex." - example: "^component1$" - - name: "--tag" - alternatives: [ "-t" ] - type: string - description: Which tag/version of the pipeline to use. - example: "0.1.0" - - name: "--registry" - alternatives: [ "-r" ] - example: ghcr.io - type: string - description: Which Docker registry to use in the Docker image name. - - name: "--organization" - alternatives: [ "-o", "--organisation" ] - example: myorganisation - type: string - description: Which organisation name to use in the Docker image name. - - name: "--target_image_source" - alternatives: [ "-tis" ] - type: string - description: Which image source to specify in the component builds. - example: https://github.com/myorganisation/myrepository - - name: "--namespace_separator" - example: "_" - type: string - description: The separator to use between the component name and namespace as the image name of a Docker container. - - name: "--nextflow_variant" - type: string - description: "[Deprecated] Which nextflow variant to use." - - name: "--max_threads" - type: integer - description: The maximum number of threads viash will use when `--parallel` during parallel tasks. - example: 8 - - name: "--config_mod" - alternatives: [ "-c" ] - type: string - multiple: true - multiple_sep: ";" - description: "Modify a viash config at runtime using a custom DSL." - - name: "--tsv" - type: file - description: Test results stored as a tabular text file. - example: .viash_test_log.tsv - direction: output - - name: "--no_cache" - alternatives: [ "-nc", "--no-cache" ] - type: boolean_true - description: Don't cache the docker build in development mode. - - name: "--log" - alternatives: [ "-l" ] - type: file - description: Test log file - example: .viash_test_log.txt - direction: output - - name: "--append" - type: boolean - default: true - description: Append to the log file? - - name: "--viash" - type: file - description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. - - name: "--verbose" - type: boolean_true - description: "Increase verbosity." - resources: - - type: bash_script - path: script.sh - test_resources: - - type: bash_script - path: run_test.sh - - path: ../../test/resources/testns/src -platforms: -- type: native diff --git a/src/viash/viash_test/run_test.sh b/src/viash/viash_test/run_test.sh deleted file mode 100644 index 95047e05a..000000000 --- a/src/viash/viash_test/run_test.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -defaults_output="defaults_test_output.txt" - -alt_output=alt_test_output.txt -alt_src="alt_src" -log=build_log.txt -tsv=output.tsv - -# 1. Run component with default arguments -# Run component -set +e -$meta_executable \ - --verbose \ - >$defaults_output - -exit_code="$?" -set -e - -[ $exit_code -ne 1 ] && echo "Expected exit code 1 but received $exit_code" && exit 1 - -# Check if defaults output exists -[[ ! -f $defaults_output ]] && echo "Default: Test output file could not be found!" && exit 1 - -# # Check if default arguments are as expected -grep -q "viash ns test --src src --parallel --config_mod .functionality.version := 'dev' --config_mod .platforms\[.type == 'docker'\].setup_strategy := 'cachedbuild' --platform docker" $defaults_output - -# 2. Run component with custom arguments -# Copy src dir -cp -r src $alt_src - -# Run component -unset exit_code -set +e -$meta_executable \ - --verbose \ - --src $alt_src \ - --platform native \ - --mode release \ - --tag rc-1 \ - --query_namespace 'testns' \ - --query_name 'ns_a' \ - --registry 'my_registry' \ - --organization 'my_organization' \ - --target_image_source 'https://github.com/viash-io/viash' \ - --namespace_separator '*~*' \ - --max_threads 8 \ - --config_mod '.functionality.version := "5.0"' \ - --log $log \ - --tsv $tsv \ - >$alt_output - -exit_code="$?" -set -e -[ $exit_code -ne 1 ] && echo "Expected exit code 0 but received $exit_code" && exit 1 - -# Check if alt output exists -[[ ! -f $alt_output ]] && echo "Alt: Test output file could not be found!" && exit 1 - -# The build log should contain an error as ns_error can't be built -grep -q "Reading file 'alt_src/ns_error/config.vsh.yaml' failed" $log - -# Check if tsv exists -[[ ! -f $tsv ]] && echo "TSV not found!" && exit 1 - -# Check if ns_add was tested -grep -q "ns_add" $tsv - -# Check if the test was succesful was tested -grep -q "SUCCESS" $tsv - -# Check if the namespace of ns_add is testns -grep -q "testns" $tsv - -echo ">>> Test finished successfully" diff --git a/src/viash/viash_test/script.sh b/src/viash/viash_test/script.sh deleted file mode 100644 index 80e35a74b..000000000 --- a/src/viash/viash_test/script.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash - -# start creating command -command_builder=( - ns test - --src "$par_src" - --parallel -) - -# check par mode -if [ "$par_mode" == "development" ]; then - echo "In development mode with 'dev'." -elif [ "$par_mode" == "integration" ]; then - echo "In integration mode with tag '$par_tag'." -elif [ "$par_mode" == "release" ]; then - echo "In RELEASE mode with tag '$par_tag'." -else - echo "Error: Not a valid mode argument '$par_mode'." - exit 1 -fi - -# check tag -if [ "$par_mode" == "development" ]; then - if [ ! -z "$par_tag" ]; then - echo "Warning: '--tag' is ignored when '--mode=$par_mode'." - fi - par_tag="dev" -fi -if [ -z "$par_tag" ]; then - echo "Error: --tag is a requirement argument when '--mode=$par_mode'." - exit 1 -fi - -# derive setup strategy -if [ "$par_mode" == "development" ]; then - if [ "$par_no_cache" == "true" ]; then - setup_strat="build" - else - setup_strat="cachedbuild" - fi -elif [ "$par_mode" == "integration" ]; then - echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." - setup_strat="ifneedbepullelsecachedbuild" -elif [ "$par_mode" == "release" ]; then - echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." - setup_strat="build" -fi - -command_builder+=( - --config_mod ".functionality.version := '$par_tag'" - --config_mod ".platforms[.type == 'docker'].setup_strategy := '$setup_strat'" -) - -# check registry and organization -if [ "$par_mode" == "development" ]; then - if [ ! -z "$par_registry" ]; then - [[ "$par_verbose" == "true" ]] && echo "Note: --par_registry is ignored when '--mode=development'." - unset par_registry - fi - - if [ ! -z "$par_organization" ]; then - [[ "$par_verbose" == "true" ]] && echo "Note: --par_organization is ignored when '--mode=development'." - unset par_organization - fi -fi - -################ COMMON PARAMS ################ - -# check viash arg -# if not specified, default par_viash to look for 'viash' on the PATH -if [ -z "$par_viash" ]; then - par_viash="viash" -fi - -# if specified, use par_max_threads as a java argument -if [ ! -z "$par_max_threads" ]; then - export JAVA_ARGS="$JAVA_ARGS -Dscala.concurrent.context.maxThreads=$par_max_threads" -fi - -# process queries -if [ ! -z "$par_query" ]; then - command_builder+=("--query" "$par_query") -fi -if [ ! -z "$par_query_namespace" ]; then - command_builder+=("--query_namespace" "$par_query_namespace") -fi -if [ ! -z "$par_query_name" ]; then - command_builder+=("--query_name" "$par_query_name") -fi - -# process config mods -if [ ! -z "$par_config_mod" ]; then - IFS=";" - for var in $par_config_mod; do - unset IFS - command_builder+=("--config_mod" "$var") - done -fi - -if [ ! -z "$par_registry" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_registry := '$par_registry'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].registry := '$par_registry'" - ) -fi - -if [ ! -z "$par_organization" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_organization := '$par_organization'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].organization := '$par_organization'" - ) -fi - -if [ ! -z "$par_namespace_separator" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].namespace_separator := '$par_namespace_separator'" - --config_mod ".platforms[.type == 'nextflow' && .variant == 'legacy'].namespace_separator := '$par_namespace_separator'" - ) -fi - -if [ ! -z "$par_target_image_source" ]; then - command_builder+=( - --config_mod ".platforms[.type == 'docker'].target_image_source := '$par_target_image_source'" - ) -fi - -if [ ! -z "$par_platform" ]; then - command_builder+=(--platform "$par_platform") -fi - -if [ "$par_append" == "true" ]; then - command_builder+=("--append") -fi - -if [ ! -z "$par_tsv" ]; then - command_builder+=(--tsv "$par_tsv") -fi - -################ RUN COMMAND ################ -[[ "$par_verbose" == "true" ]] && echo "+ $par_viash" "${command_builder[@]}" - -if [ -z "$par_log" ]; then - "$par_viash" "${command_builder[@]}" -else - [ ! -f "$par_log" ] || rm "$par_log" - "$par_viash" "${command_builder[@]}" > >(tee -a "$par_log") 2> >(tee -a "$par_log") -fi