Add schema, validation and linting (for .ga and Format 2 in Python and Java)

.gitignore

@@ -41,3 +41,4 @@ docs/_build
 
 .venv
 
+java/target

Makefile

@@ -55,7 +55,7 @@ flake8: ## check style using flake8 for current Python (faster than lint)
 	$(IN_VENV) flake8 --max-complexity 11 $(SOURCE_DIR)  $(TEST_DIR)
 
 lint: ## check style using tox and flake8 for Python 2 and Python 3
-	$(IN_VENV) tox -e py27-lint && tox -e py34-lint
+	$(IN_VENV) tox -e py27-lint && tox -e py35-lint
 
 lint-readme: ## check README formatting for PyPI
 	$(IN_VENV) python setup.py check -r -s

build_schema.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -x
+set -e
+
+PROJECT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+
+# Requires schema-salad-doc that recognizes --brandstyle and --brandinverse
+for schema in "v19.09";
+do
+    cd schema/"$schema";
+    python_schema_name=${schema//./_}
+    schema-salad-tool --codegen python workflow.yml > "${PROJECT_DIRECTORY}/gxformat2/schema/${python_schema_name}.py"
+
+    out="../${schema}.html"
+    schema-salad-doc \
+        --brandstyle '<link rel="stylesheet" href="https://jamestaylor.org/galaxy-bootstrap/galaxy_bootstrap.css">' \
+        --brandinverse \
+        --brand '<img src="icon.png" />' \
+        --only "https://galaxyproject.org/gxformat2/${schema}#WorkflowDoc" \
+        --only "https://galaxyproject.org/gxformat2/${schema}#GalaxyWorkflow" \
+        workflow.yml > "$out"
+
+    java_package="${PROJECT_DIRECTORY}/java"
+    schema-salad-tool \
+        --codegen java \
+        --codegen-target "$java_package" \
+        --codegen-examples examples \
+        workflow.yml
+    cd "$java_package"
+    mvn test
+    mvn javadoc:javadoc
+    cd "${PROJECT_DIRECTORY}"
+done

dev-requirements.txt

@@ -1,3 +1,6 @@
+# Optional dependencies
+schema-salad
+
 # For testing
 tox
 nose

gxformat2/lint.py

@@ -0,0 +1,110 @@
+"""Workflow linting entry point - main script."""
+import os
+import sys
+
+from gxformat2._yaml import ordered_load
+from gxformat2.linting import LintContext
+
+EXIT_CODE_SUCCESS = 0
+EXIT_CODE_LINT_FAILED = 1
+EXIT_CODE_FORMAT_ERROR = 2
+EXIT_CODE_FILE_PARSE_FAILED = 3
+
+LINT_FAILED_NO_OUTPUTS = "Workflow contained no outputs"
+LINT_FAILED_OUTPUT_NO_LABEL = "Workflow contained output without a label"
+
+
+def ensure_key(lint_context, has_keys, key, has_class=None, has_value=None):
+    if key not in has_keys:
+        lint_context.error("expected to find key [{key}] but absent", key=key)
+        return None
+
+    value = has_keys[key]
+    return ensure_key_has_value(lint_context, has_keys, key, value, has_class=has_class, has_value=has_value)
+
+
+def ensure_key_if_present(lint_context, has_keys, key, default=None, has_class=None):
+    if key not in has_keys:
+        return default
+
+    value = has_keys[key]
+    return ensure_key_has_value(lint_context, has_keys, key, value, has_class=has_class, has_value=None)
+
+
+def ensure_key_has_value(lint_context, has_keys, key, value, has_class=None, has_value=None):
+    if has_class is not None and not isinstance(value, has_class):
+        lint_context.error("expected value [{value}] with key [{key}] to be of class {clazz}", key=key, value=value, clazz=has_class)
+    if has_value is not None and value != has_value:
+        lint_context.error("expected value [{value}] with key [{key}] to be {expected_value}", key=key, value=value, expected_value=has_value)
+    return value
+
+
+def lint_ga(lint_context, workflow_dict, path=None):
+    """Lint a native/legacy style Galaxy workflow and populate the corresponding LintContext."""
+    ensure_key(lint_context, workflow_dict, "format-version", has_value="0.1")
+    ensure_key(lint_context, workflow_dict, "a_galaxy_workflow", has_value="true")
+
+    native_steps = ensure_key(lint_context, workflow_dict, "steps", has_class=dict) or {}
+
+    found_outputs = False
+    found_output_without_label = False
+    for order_index_str, step in native_steps.items():
+        if not order_index_str.isdigit():
+            lint_context.error("expected step_key to be integer not [{value}]", value=order_index_str)
+
+        workflow_outputs = ensure_key_if_present(lint_context, step, "workflow_outputs", default=[], has_class=list)
+        for workflow_output in workflow_outputs:
+            found_outputs = True
+
+            print(workflow_output)
+            if not workflow_output.get("label"):
+                found_output_without_label = True
+
+        step_type = step.get("type")
+        if step_type == "subworkflow":
+            subworkflow = ensure_key(lint_context, step, "subworkflow", has_class=dict)
+            lint_ga(lint_context, subworkflow)
+
+    if not found_outputs:
+        lint_context.warn(LINT_FAILED_NO_OUTPUTS)
+
+    if found_output_without_label:
+        lint_context.warn(LINT_FAILED_OUTPUT_NO_LABEL)
+
+
+def lint_format2(lint_context, workflow_dict, path=None):
+    """Lint a Format 2 Galaxy workflow and populate the corresponding LintContext."""
+    from gxformat2.schema.v19_09 import load_document
+    from schema_salad.exceptions import SchemaSaladException
+    try:
+        load_document("file://" + os.path.normpath(path))
+    except SchemaSaladException as e:
+        lint_context.error("Validation failed " + str(e))
+
+
+def main(argv):
+    """Script entry point for linting workflows."""
+    path = argv[1]
+    with open(path, "r") as f:
+        try:
+            workflow_dict = ordered_load(f)
+        except Exception:
+            return EXIT_CODE_FILE_PARSE_FAILED
+    workflow_class = workflow_dict.get("class")
+    lint_func = lint_format2 if workflow_class == "GalaxyWorkflow" else lint_ga
+    lint_context = LintContext()
+    lint_func(lint_context, workflow_dict, path=path)
+    lint_context.print_messages()
+    if lint_context.found_errors:
+        return EXIT_CODE_FORMAT_ERROR
+    elif lint_context.found_warns:
+        return EXIT_CODE_LINT_FAILED
+    else:
+        return EXIT_CODE_SUCCESS
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
+
+
+__all__ = ('main', 'lint_format2', 'lint_ga')

gxformat2/linting.py

@@ -0,0 +1,58 @@
+"""Generic utilities for linting.
+
+Largely derived in large part from galaxy.tool_util.lint.
+"""
+LEVEL_ALL = "all"
+LEVEL_WARN = "warn"
+LEVEL_ERROR = "error"
+
+
+class LintContext(object):
+    """Track running status (state) of linting."""
+
+    def __init__(self, level=LEVEL_WARN):
+        """Create LintContext with specified 'level' (currently unused)."""
+        self.level = level
+        self.found_errors = False
+        self.found_warns = False
+
+        # self.valid_messages = []
+        # self.info_messages = []
+        self.warn_messages = []
+        self.error_messages = []
+
+    def __handle_message(self, message_list, message, *args, **kwds):
+        if kwds or args:
+            message = message.format(*args, **kwds)
+        message_list.append(message)
+
+    # def valid(self, message, *args, **kwds):
+    #     self.__handle_message(self.valid_messages, message, *args, **kwds)
+
+    # def info(self, message, *args, **kwds):
+    #     self.__handle_message(self.info_messages, message, *args, **kwds)
+
+    def error(self, message, *args, **kwds):
+        """Track a linting error - a serious problem with the artifact preventing execution."""
+        self.__handle_message(self.error_messages, message, *args, **kwds)
+
+    def warn(self, message, *args, **kwds):
+        """Track a linting warning - a deviation from best practices."""
+        self.__handle_message(self.warn_messages, message, *args, **kwds)
+
+    def print_messages(self):
+        """Print error messages and update state at the end of linting."""
+        for message in self.error_messages:
+            self.found_errors = True
+            print(".. ERROR: %s" % message)
+
+        if self.level != LEVEL_ERROR:
+            for message in self.warn_messages:
+                self.found_warns = True
+                print(".. WARNING: %s" % message)
+
+        if self.level == LEVEL_ALL:
+            for message in self.info_messages:
+                print(".. INFO: %s" % message)
+            for message in self.valid_messages:
+                print(".. CHECK: %s" % message)

gxformat2/schema/__init__.py

@@ -0,0 +1,5 @@
+"""Parsers for workflows dervied from schema-salad descriptions and codegen.
+
+Python files in this package probably shouldn't be modified manually. See
+build_schema.sh for more information.
+"""