From e9c91bbf1a7468d0a886a185dc0b1a7f587a6fa8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 2 Dec 2019 19:31:53 -0500 Subject: [PATCH] Lint with messages now. --- gxformat2/lint.py | 82 ++++++++++++------- gxformat2/linting.py | 49 +++++++++++ .../gxformat2/Format2Linter.java | 5 +- .../gxformat2/GalaxyWorkflowLinter.java | 7 +- .../org/galaxyproject/gxformat2/Lint.java | 18 +++- .../galaxyproject/gxformat2/LintContext.java | 52 ++++++++++++ .../galaxyproject/gxformat2/LintUtils.java | 51 ++++++++++++ .../galaxyproject/gxformat2/NativeLinter.java | 68 +++++++-------- 8 files changed, 260 insertions(+), 72 deletions(-) create mode 100644 gxformat2/linting.py create mode 100644 java/src/main/java/org/galaxyproject/gxformat2/LintContext.java create mode 100644 java/src/main/java/org/galaxyproject/gxformat2/LintUtils.java diff --git a/gxformat2/lint.py b/gxformat2/lint.py index a279ea6..268ed19 100644 --- a/gxformat2/lint.py +++ b/gxformat2/lint.py @@ -2,68 +2,85 @@ import sys from gxformat2._yaml import ordered_load +from gxformat2.linting import LintContext EXIT_CODE_SUCCESS = 0 EXIT_CODE_LINT_FAILED = 1 EXIT_CODE_FORMAT_ERROR = 2 EXIT_CODE_FILE_PARSE_FAILED = 3 +LINT_FAILED_NO_OUTPUTS = "Workflow contained no outputs" +LINT_FAILED_OUTPUT_NO_LABEL = "Workflow contained output without a label" -def lint_ga(workflow_dict, path=None): - if workflow_dict.get("format-version") != "0.1": - return EXIT_CODE_FORMAT_ERROR - if workflow_dict.get("a_galaxy_workflow") != "true": - return EXIT_CODE_FORMAT_ERROR - native_steps = workflow_dict.get("steps") - if not native_steps or not isinstance(native_steps, dict): - return EXIT_CODE_FORMAT_ERROR +def ensure_key(lint_context, has_keys, key, has_class=None, has_value=None): + if key not in has_keys: + lint_context.error("expected to find key [{key}] but absent", key=key) + return None + + value = has_keys[key] + return ensure_key_has_value(lint_context, has_keys, key, value, has_class=has_class, has_value=has_value) + + +def ensure_key_if_present(lint_context, has_keys, key, default=None, has_class=None): + if key not in has_keys: + return default + + value = has_keys[key] + return ensure_key_has_value(lint_context, has_keys, key, value, has_class=has_class, has_value=None) + + +def ensure_key_has_value(lint_context, has_keys, key, value, has_class=None, has_value=None): + if has_class is not None and not isinstance(value, has_class): + lint_context.error("expected value [{value}] with key [{key}] to be of class {clazz}", key=key, value=value, clazz=has_class) + if has_value is not None and value != has_value: + lint_context.error("expected value [{value}] with key [{key}] to be {expected_value}", key=key, value=value, expected_value=has_value) + return value + + +def lint_ga(lint_context, workflow_dict, path=None): + ensure_key(lint_context, workflow_dict, "format-version", has_value="0.1") + ensure_key(lint_context, workflow_dict, "a_galaxy_workflow", has_value="true") + + native_steps = ensure_key(lint_context, workflow_dict, "steps", has_class=dict) or {} found_outputs = False found_output_without_label = False - for order_index_str, step in native_steps.items(): if not order_index_str.isdigit(): - return EXIT_CODE_FORMAT_ERROR + lint_context.error("expected step_key to be integer not [{value}]", value=order_index_str) - for workflow_output in step.get("workflow_outputs", []): + workflow_outputs = ensure_key_if_present(lint_context, step, "workflow_outputs", default=[], has_class=list) + for workflow_output in workflow_outputs: found_outputs = True + print(workflow_output) if not workflow_output.get("label"): found_output_without_label = True step_type = step.get("type") if step_type == "subworkflow": - subworkflow = step.get("subworkflow") - if subworkflow and not isinstance(subworkflow, dict): - return EXIT_CODE_FORMAT_ERROR - lint_subworkflow_ret = lint_ga(subworkflow) - if lint_subworkflow_ret != 0: - return lint_subworkflow_ret + subworkflow = ensure_key(lint_context, step, "subworkflow", has_class=dict) + lint_ga(lint_context, subworkflow) if not found_outputs: - return EXIT_CODE_LINT_FAILED + lint_context.warn(LINT_FAILED_NO_OUTPUTS) if found_output_without_label: - return EXIT_CODE_LINT_FAILED - - return EXIT_CODE_SUCCESS + lint_context.warn(LINT_FAILED_OUTPUT_NO_LABEL) -def lint_format2(workflow_dict, path=None): +def lint_format2(lint_context, workflow_dict, path=None): from gxformat2.schema.v19_09 import load_document from schema_salad.exceptions import SchemaSaladException try: load_document("file://" + os.path.normpath(path)) except SchemaSaladException as e: - print(e) - return EXIT_CODE_FORMAT_ERROR + lint_context.error("Validation failed " + str(e)) # Lint for outputs... if not workflow_dict.get("outputs", None): - return EXIT_CODE_LINT_FAILED - - return EXIT_CODE_SUCCESS + lint_context.warn(LINT_FAILED_NO_OUTPUTS) def main(argv): @@ -75,8 +92,15 @@ def main(argv): return EXIT_CODE_FILE_PARSE_FAILED workflow_class = workflow_dict.get("class") lint_func = lint_format2 if workflow_class == "GalaxyWorkflow" else lint_ga - exit_code = lint_func(workflow_dict, path=path) - return exit_code + lint_context = LintContext() + lint_func(lint_context, workflow_dict, path=path) + lint_context.print_messages() + if lint_context.found_errors: + return EXIT_CODE_FORMAT_ERROR + elif lint_context.found_warns: + return EXIT_CODE_LINT_FAILED + else: + return EXIT_CODE_SUCCESS if __name__ == "__main__": diff --git a/gxformat2/linting.py b/gxformat2/linting.py new file mode 100644 index 0000000..ecd9bad --- /dev/null +++ b/gxformat2/linting.py @@ -0,0 +1,49 @@ +LEVEL_ALL = "all" +LEVEL_WARN = "warn" +LEVEL_ERROR = "error" + + +class LintContext(object): + + def __init__(self, level=LEVEL_WARN): + self.level = level + self.found_errors = False + self.found_warns = False + + self.valid_messages = [] + self.info_messages = [] + self.warn_messages = [] + self.error_messages = [] + + def __handle_message(self, message_list, message, *args, **kwds): + if kwds or args: + message = message.format(*args, **kwds) + message_list.append(message) + + def valid(self, message, *args, **kwds): + self.__handle_message(self.valid_messages, message, *args, **kwds) + + def info(self, message, *args, **kwds): + self.__handle_message(self.info_messages, message, *args, **kwds) + + def error(self, message, *args, **kwds): + self.__handle_message(self.error_messages, message, *args, **kwds) + + def warn(self, message, *args, **kwds): + self.__handle_message(self.warn_messages, message, *args, **kwds) + + def print_messages(self): + for message in self.error_messages: + self.found_errors = True + print(".. ERROR: %s" % message) + + if self.level != LEVEL_ERROR: + for message in self.warn_messages: + self.found_warns = True + print(".. WARNING: %s" % message) + + if self.level == LEVEL_ALL: + for message in self.info_messages: + print(".. INFO: %s" % message) + for message in self.valid_messages: + print(".. CHECK: %s" % message) diff --git a/java/src/main/java/org/galaxyproject/gxformat2/Format2Linter.java b/java/src/main/java/org/galaxyproject/gxformat2/Format2Linter.java index 20d35b3..1220762 100644 --- a/java/src/main/java/org/galaxyproject/gxformat2/Format2Linter.java +++ b/java/src/main/java/org/galaxyproject/gxformat2/Format2Linter.java @@ -5,12 +5,11 @@ import org.galaxyproject.gxformat2.v19_09.utils.ValidationException; public class Format2Linter implements GalaxyWorkflowLinter { - public int lint(final Map workflow) { + public void lint(final LintContext lintContext, final Map workflow) { try { RootLoader.loadDocument(workflow); } catch (ValidationException e) { - return EXIT_CODE_FORMAT_ERROR; + lintContext.error("Validation failed " + e.toString()); } - return EXIT_CODE_SUCCESS; } } diff --git a/java/src/main/java/org/galaxyproject/gxformat2/GalaxyWorkflowLinter.java b/java/src/main/java/org/galaxyproject/gxformat2/GalaxyWorkflowLinter.java index b4e9d74..eccedda 100644 --- a/java/src/main/java/org/galaxyproject/gxformat2/GalaxyWorkflowLinter.java +++ b/java/src/main/java/org/galaxyproject/gxformat2/GalaxyWorkflowLinter.java @@ -3,10 +3,5 @@ import java.util.Map; public interface GalaxyWorkflowLinter { - public static int EXIT_CODE_SUCCESS = 0; - public static int EXIT_CODE_LINT_FAILED = 1; - public static int EXIT_CODE_FORMAT_ERROR = 2; - public static int EXIT_CODE_FILE_PARSE_FAILED = 3; - - public int lint(final Map workflow); + public void lint(final LintContext lintContext, final Map workflow); } diff --git a/java/src/main/java/org/galaxyproject/gxformat2/Lint.java b/java/src/main/java/org/galaxyproject/gxformat2/Lint.java index 447700b..8d0c795 100644 --- a/java/src/main/java/org/galaxyproject/gxformat2/Lint.java +++ b/java/src/main/java/org/galaxyproject/gxformat2/Lint.java @@ -7,13 +7,17 @@ import org.yaml.snakeyaml.Yaml; public class Lint { + public static int EXIT_CODE_SUCCESS = 0; + public static int EXIT_CODE_LINT_FAILED = 1; + public static int EXIT_CODE_FORMAT_ERROR = 2; + public static int EXIT_CODE_FILE_PARSE_FAILED = 3; + public static void main(String[] args) throws Exception { final int exitCode = lint(args); System.exit(exitCode); } public static int lint(final String[] args) throws Exception { - final Path path = Paths.get(args[0]); final String workflowContents = new String(Files.readAllBytes(path), "UTF8"); final Yaml yaml = new Yaml(); @@ -25,7 +29,17 @@ public static int lint(final String[] args) throws Exception { } else { linter = new NativeLinter(); } - final int exitCode = linter.lint(object); + final LintContext lintContext = new LintContext(); + linter.lint(lintContext, object); + lintContext.printMessages(); + int exitCode; + if (lintContext.getFoundErrors()) { + exitCode = EXIT_CODE_FORMAT_ERROR; + } else if (lintContext.getFoundWarns()) { + exitCode = EXIT_CODE_LINT_FAILED; + } else { + exitCode = EXIT_CODE_SUCCESS; + } return exitCode; } } diff --git a/java/src/main/java/org/galaxyproject/gxformat2/LintContext.java b/java/src/main/java/org/galaxyproject/gxformat2/LintContext.java new file mode 100644 index 0000000..41b0c84 --- /dev/null +++ b/java/src/main/java/org/galaxyproject/gxformat2/LintContext.java @@ -0,0 +1,52 @@ +package org.galaxyproject.gxformat2; + +import java.util.ArrayList; +import java.util.List; + +public class LintContext { + private boolean foundErrors = false; + private boolean foundWarns = false; + + private List validMessages = new ArrayList(); + private List infoMessages = new ArrayList(); + private List warnMessages = new ArrayList(); + private List errorMessages = new ArrayList(); + + LintContext() {} + + boolean getFoundErrors() { + return this.foundErrors; + } + + boolean getFoundWarns() { + return this.foundWarns; + } + + void valid(String message, Object... args) { + this.validMessages.add(String.format(message, args)); + } + + void info(String message, Object... args) { + this.infoMessages.add(String.format(message, args)); + } + + void error(String message, Object... args) { + this.errorMessages.add(String.format(message, args)); + } + + void warn(String message, Object... args) { + this.warnMessages.add(String.format(message, args)); + } + + void printMessages() { + for (final String message : this.errorMessages) { + this.foundErrors = true; + System.out.print(".. ERROR " + message); + } + + for (final String message : this.warnMessages) { + this.foundWarns = true; + System.out.print(".. WARNING " + message); + } + } +} diff --git a/java/src/main/java/org/galaxyproject/gxformat2/LintUtils.java b/java/src/main/java/org/galaxyproject/gxformat2/LintUtils.java new file mode 100644 index 0000000..e8bf276 --- /dev/null +++ b/java/src/main/java/org/galaxyproject/gxformat2/LintUtils.java @@ -0,0 +1,51 @@ +package org.galaxyproject.gxformat2; + +import java.util.Map; + +class LintUtils { + static T ensureKey( + LintContext lintContext, Object hasKeys, String key, Class hasClass, Object hasValue) { + if (!(hasKeys instanceof Map)) { + lintContext.error("expected [%s] to be a dictionary type", hasKeys); + return null; + } + final Map map = (Map) hasKeys; + if (!map.containsKey(key)) { + lintContext.error("expected to have key [%s] but absent", key); + } + final Object value = map.get(key); + return ensureKeyHasValue(lintContext, map, key, value, hasClass, hasValue); + } + + static T ensureKeyIfPresent( + LintContext lintContext, Object hasKeys, String key, T defaultValue, Class hasClass) { + if (!(hasKeys instanceof Map)) { + lintContext.error("expected [%s] to be a dictionary type", hasKeys); + return null; + } + final Map map = (Map) hasKeys; + if (!map.containsKey(key)) { + return defaultValue; + } + final Object value = map.get(key); + return ensureKeyHasValue(lintContext, map, key, value, hasClass, null); + } + + static T ensureKeyHasValue( + LintContext lintContext, + Map hasKeys, + String key, + Object value, + Class hasClass, + Object hasValue) { + if (!hasClass.isInstance(value)) { + lintContext.error( + "expected value [%s] with key [%s] to be of class %s", key, value, hasClass); + return null; + } + if (hasValue != null && !hasValue.equals(value)) { + lintContext.error("expected value [%s] with key [%s] to be %s", key, value, hasValue); + } + return (T) value; + } +} diff --git a/java/src/main/java/org/galaxyproject/gxformat2/NativeLinter.java b/java/src/main/java/org/galaxyproject/gxformat2/NativeLinter.java index d6235d3..9bd3c31 100644 --- a/java/src/main/java/org/galaxyproject/gxformat2/NativeLinter.java +++ b/java/src/main/java/org/galaxyproject/gxformat2/NativeLinter.java @@ -1,45 +1,40 @@ package org.galaxyproject.gxformat2; +import java.util.ArrayList; import java.util.List; import java.util.Map; public class NativeLinter implements GalaxyWorkflowLinter { - public int lint(final Map workflow) { - if (!"0.1".equals(workflow.get("format-version"))) { - return EXIT_CODE_FORMAT_ERROR; - } - if (!"true".equals(workflow.get("a_galaxy_workflow"))) { - return EXIT_CODE_FORMAT_ERROR; - } + public static String LINT_FAILED_NO_OUTPUTS = "Workflow contained no outputs"; + public static String LINT_FAILED_OUTPUT_NO_LABEL = "Workflow contained output without a label"; - Object nativeSteps = workflow.get("steps"); - if (!(nativeSteps instanceof Map)) { - return EXIT_CODE_FORMAT_ERROR; - } + public void lint(final LintContext lintContext, final Map workflow) { + LintUtils.ensureKey(lintContext, workflow, "format-version", String.class, "0.1"); + LintUtils.ensureKey(lintContext, workflow, "a_galaxy_workflow", String.class, "true"); + + final Map steps = + (Map) LintUtils.ensureKey(lintContext, workflow, "steps", Map.class, null); boolean foundOutputs = false; boolean foundOutputsWithoutLabel = false; - Map steps = (Map) nativeSteps; - for (Map.Entry stepEntry : steps.entrySet()) { - final String orderIndexStr = stepEntry.getKey(); - try { - final int orderIndex = Integer.parseInt(orderIndexStr); - } catch (NumberFormatException e) { - return EXIT_CODE_FORMAT_ERROR; - } - if (!(stepEntry.getValue() instanceof Map)) { - return EXIT_CODE_FORMAT_ERROR; - } - Map step = (Map) stepEntry.getValue(); - Object workflowOutputsObject = step.get("workflow_outputs"); - if (workflowOutputsObject != null) { - if (!(workflowOutputsObject instanceof List)) { - return EXIT_CODE_FORMAT_ERROR; + if (steps != null) { + for (Map.Entry stepEntry : steps.entrySet()) { + final String orderIndexStr = stepEntry.getKey(); + try { + final int orderIndex = Integer.parseInt(orderIndexStr); + } catch (NumberFormatException e) { + lintContext.error("expected step_key to be integer not [%s]", orderIndexStr); } - final List workflowOutputs = (List) workflowOutputsObject; + if (!(stepEntry.getValue() instanceof Map)) { + lintContext.error("expected step value to be Map not [%s]", stepEntry.getValue()); + } + Map step = (Map) stepEntry.getValue(); + List workflowOutputs = + LintUtils.ensureKeyIfPresent( + lintContext, step, "workflow_outputs", new ArrayList(), List.class); for (Object workflowOutputObject : workflowOutputs) { foundOutputs = true; if (!(workflowOutputObject instanceof Map)) { - return EXIT_CODE_FORMAT_ERROR; + lintContext.error("Not a map"); } final Map workflowOutput = (Map) workflowOutputObject; final String label = workflowOutput.get("label"); @@ -47,14 +42,23 @@ public int lint(final Map workflow) { foundOutputsWithoutLabel = true; } } + final String stepType = + LintUtils.ensureKeyIfPresent(lintContext, step, "type", "tool", String.class); + assert stepType != null; + if (stepType != null && stepType.equals("subworkflow")) { + final Map subworkflow = + (Map) + LintUtils.ensureKey(lintContext, step, "subworkflow", Map.class, null); + assert subworkflow != null; + lint(lintContext, subworkflow); + } } } if (!foundOutputs) { - return EXIT_CODE_LINT_FAILED; + lintContext.warn(LINT_FAILED_NO_OUTPUTS); } if (foundOutputsWithoutLabel) { - return EXIT_CODE_LINT_FAILED; + lintContext.warn(LINT_FAILED_OUTPUT_NO_LABEL); } - return EXIT_CODE_SUCCESS; } }