diff --git a/tools/.dockerignore b/tools/.dockerignore deleted file mode 100644 index 93dc53fb01..0000000000 --- a/tools/.dockerignore +++ /dev/null @@ -1,9 +0,0 @@ -unit_test/**/* -unit_test -examples/**/* -examples -tests -tests/**/* -**/*.json -**/*.tar -**/*.gz \ No newline at end of file diff --git a/tools/Makefiles/Colors.mk b/tools/Makefiles/Colors.mk deleted file mode 100644 index 0563a63270..0000000000 --- a/tools/Makefiles/Colors.mk +++ /dev/null @@ -1,7 +0,0 @@ -Q = @ -REPO = $(shell grep url .git/config) -BOLD = $(shell tput bold) -UNDERLINE = $(shell tput smul) -NORMAL = $(shell tput sgr0) -RED = $(shell tput setaf 1) -YELLOW = $(shell tput setaf 3) diff --git a/tools/Makefiles/Helpers.mk b/tools/Makefiles/Helpers.mk deleted file mode 100755 index f2852c2157..0000000000 --- a/tools/Makefiles/Helpers.mk +++ /dev/null @@ -1,23 +0,0 @@ -menu: - $(Q)echo "" - $(Q)echo "$(BOLD)Plugin main targets:$(NORMAL)" - $(Q)echo " $(RED)default$(NORMAL) - Builds the plugin and creates a tarball (image + tarball targets)" - $(Q)echo " $(RED)image$(NORMAL) - Build a Docker image of the current plugin directory" - $(Q)echo " $(RED)tarball$(NORMAL) - Create a plugin tarball of the current directory" - $(Q)echo " $(RED)regenerate$(NORMAL) - Regenerate the plugin schema from plugin.spec.yaml" - $(Q)echo "$(BOLD)Plugin helper targets:$(NORMAL) (depends on installed tools: ../tools/)" - $(Q)echo " $(RED)menu$(NORMAL) - This menu" - $(Q)echo " $(RED)validate$(NORMAL) - Runs the plugin's files through installed validators" - $(Q)echo " $(RED)update-tools$(NORMAL) - Automatically updates all your plugin tooling" - -validate: - $(info [$(YELLOW)*$(NORMAL)] Running validators) - @python3 -m pip install --upgrade insightconnect-integrations-validators > /dev/null && icon-validate . - @test -x ../tools/check_spec.py && ../tools/check_spec.py ./plugin.spec.yaml || true - @test -x ../tools/mdl.sh && ../tools/mdl.sh || true - @test -x ../tools/flake8.sh && ../tools/flake8.sh || true - @test -x ../tools/bandit.sh && ../tools/bandit.sh || true - @test -x ../tools/misspell.sh && ../tools/misspell.sh || true - -update-tools: - ../tools/update-tools.sh diff --git a/tools/bandit.sh b/tools/bandit.sh deleted file mode 100755 index b66262d5bb..0000000000 --- a/tools/bandit.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -BOLD=$(tput bold) -UNDERLINE=$(tput smul) -NORMAL=$(tput sgr0) -RED=$(tput setaf 1) -YELLOW=$(tput setaf 3) - - -is_python() { - if [[ -f "setup.py" ]]; then - return 0 - fi - return 1 -} - -is_pyflakes() { - if [[ -x $(which bandit || echo "None") ]]; then - return 0 - fi - printf "\n[${YELLOW}*${NORMAL}] ${UNDERLINE}For Python security tests, install bandit:${NORMAL} pip install --upgrade bandit\n" && return 1 -} - - -main() { - is_python || exit 0 - is_pyflakes || exit 0 - printf "\n[${YELLOW}*${NORMAL}] ${BOLD}Validating python files for security vulnerabilities...${NORMAL}\n" - bandit -r . - result=$? - [[ $result -eq 0 ]] && printf "[${YELLOW}SUCCESS${NORMAL}] Passes bandit security checks\n" || printf "[${RED}FAIL${NORMAL}] Fails bandit security checks\n" - return 0 # Make make happy :) -} - -main diff --git a/tools/check_spec.py b/tools/check_spec.py deleted file mode 100755 index 4c3b1cab89..0000000000 --- a/tools/check_spec.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -import subprocess # noqa: B404 -import sys -import logging - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - -RED = "\033[31m" -YELLOW = "\033[33m" -BOLD = "\033[1m" -CEND = "\033[0m" - - -def check_args(args): - """check_args ensures we are running the script with the right number of arguments""" - if len(args) != 2: - print("./check_spec.py ") - sys.exit(0) - - -def must_exec(cmd): - """must_exec ensures the executed commands are successful""" - print("") - print("[" + YELLOW + "*" + CEND + "]" + " " + BOLD + "Validating spec with js-yaml" + CEND) - exit_code = 0 - try: - subprocess.check_output(cmd) # noqa: B603 - exit_code = 0 - print("[" + YELLOW + "SUCCESS" + CEND + "]" + " Passes js-yaml spec check" + "\n") - except OSError: - print("[" + RED + "FAIL" + CEND + "]" + " js-yaml is not installed, try: node install npm") - sys.exit(1) - except subprocess.CalledProcessError as ex: - exit_code = ex.returncode - print("[" + RED + "FAIL" + CEND + "]" + " Failed js-yaml spec check") - sys.exit(1) - - -def validate(spec_path): - """validate validates a yaml file at the spec_path""" - cmd = ["js-yaml", spec_path] - must_exec(cmd) - - -if __name__ == "__main__": - check_args(sys.argv) - validate(sys.argv[1]) diff --git a/tools/check_specs.py b/tools/check_specs.py deleted file mode 100755 index 8bda272887..0000000000 --- a/tools/check_specs.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -import logging -import os -import sys - -from check_spec import validate - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - -BASE_DIR = os.path.dirname(os.path.realpath(__file__)) + "/.." - - -def get_plugin_spec(spec_dir): - """get_plugin_spec checks if there is a spec file in the dir and returns the path""" - if not os.path.isdir(spec_dir): - return None - - spec_path = os.path.join(BASE_DIR, spec_dir, "plugin.spec.yaml") - if not os.path.exists(spec_path): - return None - - return os.path.abspath(spec_path) - - -if __name__ == "__main__": - for directory in os.listdir(BASE_DIR): - spec = get_plugin_spec(directory) - if spec: - validate(spec) diff --git a/tools/flake8.sh b/tools/flake8.sh deleted file mode 100755 index 16cfde55c3..0000000000 --- a/tools/flake8.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -BOLD=$(tput bold) -UNDERLINE=$(tput smul) -NORMAL=$(tput sgr0) -RED=$(tput setaf 1) -YELLOW=$(tput setaf 3) -# List of codes to ignore -IGNORE="E501,E126,E121" - - -is_python() { - if [[ -f "setup.py" ]]; then - return 0 - fi - return 1 -} - -is_flake8() { - if [[ -x $(which flake8 || echo "None") ]]; then - return 0 - fi - printf "\n[${YELLOW}*${NORMAL}] ${UNDERLINE}For Python linting, install flake8:${NORMAL} pip install --upgrade flake8\n" && return 1 -} - - -main() { - is_python || exit 0 - is_flake8 || exit 0 - printf "\n[${YELLOW}*${NORMAL}] ${BOLD}Validating python files for style...${NORMAL}\n" - flake8 --ignore=${IGNORE} $(find . -type f -name "*.py" | egrep -v '__init__.py|schema.py') - result=$? - [[ $result -eq 0 ]] && printf "[${YELLOW}SUCCESS${NORMAL}] Passes flake8 linting\n" || printf "[${RED}FAIL${NORMAL}] Fails flake8 linting\n" - return 0 # Make make happy :) -} - -main diff --git a/tools/generate_custom_dictionary.py b/tools/generate_custom_dictionary.py deleted file mode 100644 index 30f5bff3f5..0000000000 --- a/tools/generate_custom_dictionary.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -import contextlib -import json -import gzip -import os -import re -from collections import Counter -from nltk.tag import pos_tag -from nltk.tokenize import RegexpTokenizer - - -def load_files(dirname): - files = list() - for dirpath, dirnames, filenames in os.walk(dirname): - for filename in [f for f in filenames if f.endswith(".md") or f.endswith(".yaml")]: - print("found " + os.path.join(dirpath, filename)) - files.append(os.path.join(dirpath, filename)) - return files - - -@contextlib.contextmanager -def load_file(filename, encoding="utf-8"): - if filename[-3:].lower() == ".gz": - with gzip.open(filename, mode="rt", encoding=encoding) as file: - yield file - else: - with open(filename, mode="r", encoding=encoding) as file: - yield file - - -def export_word_frequency(filepath, word_frequency): - with open(filepath, "w") as f: - json.dump(word_frequency, f, indent="", sort_keys=True, ensure_ascii=False) - - -def build_word_frequency(dirpath, output_path): - word_frequency = Counter() - tok = RegexpTokenizer(r"\w+") - - rows = 0 - - files = load_files(dirpath) - - for filepath in files: - with load_file(filepath, "utf-8") as file: - for line in file: - line = re.sub("[^0-9a-zA-Z]+", " ", line) - parts = tok.tokenize(line) - tagged_sent = pos_tag(parts) - words = [word[0].lower() for word in tagged_sent if word[0] and word[0][0].isalpha()] - if words: - word_frequency.update(words) - - rows += 1 - - if rows % 100000 == 0: - print("completed: {} rows".format(rows)) - - print("completed: {} rows".format(rows)) - export_word_frequency(output_path, word_frequency) - print("exported custom dictionary to {}".format(output_path)) - return word_frequency - - -def _parse_args(): - import argparse - - parser = argparse.ArgumentParser(description="Build a new custom dictionary for an InsightConnect Plugin") - parser.add_argument("-p", "--path", help="The path to the plugin to build a custom dictionary for") - - args = parser.parse_args() - - # validate that we have a path, if needed! - if not args.path: - raise Exception("A path is required to generate a custom dictionary") - - if args.path: - args.path = os.path.abspath(os.path.realpath(args.path)) - if not os.path.exists(args.path): - raise FileNotFoundError("A valid path is required to generate a custom dictionary") - - return args - - -if __name__ == "__main__": - args = _parse_args() - script_path = os.path.dirname(os.path.abspath(__file__)) - export_path = os.path.abspath("{}/custom_dict.json".format(args.path)) - word_frequency = build_word_frequency(args.path, export_path) diff --git a/tools/grammar.py b/tools/grammar.py deleted file mode 100755 index 061c5d3679..0000000000 --- a/tools/grammar.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -import yaml -import os, sys -import logging -import requests - -HEADERS = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"} -URL = "https://languagetool.org/api/v2/check" -RED = "\033[31m" -YELLOW = "\033[33m" -BOLD = "\033[1m" -CEND = "\033[0m" - - -def read_plugin_spec(path): - if not os.path.isfile(path): - return None - f = open(path, "r") - return yaml.safe_load(f) - - -def check_args(args): - argc = len(args) - if argc != 3: - print("./grammar.py [print|check]") - sys.exit(0) - - -def is_yaml(data): - if type(data) is None: - logging.error("Error: %s is not a file!", path) - sys.exit(1) - - -def get_text(data: dict) -> list: - # YAML sections to check - sections = ["actions", "triggers", "connection"] - # List of checked results - sentences = [] - - # Iterate over descriptions - for section in sections: - if section in data: - for top_level in data[section]: - for description in data[section][top_level]: - if isinstance(data[section][top_level][description], str): - sentences.append(data[section][top_level][description]) - if isinstance(data[section][top_level][description], dict): - for bottom_level in data[section][top_level][description]: - sentences.append(data[section][top_level][description][bottom_level]["description"]) - - # Get plugin's description - if "description" in data: - sentences.append(data["description"]) - - return sentences - - -check_args(sys.argv) -path = sys.argv[1] -arg = sys.argv[2] -plugin = path.split("/")[0] -data = read_plugin_spec(path) -is_yaml(data) -sentences = get_text(data) - -if arg == "print": - for line in sentences: - print(line) - -if arg == "check": - - for sentence in sentences: - - try: - data = {"text": sentence, "language": "en-US", "enabledOnly": "false"} - response = requests.post(URL, data=data, headers=HEADERS) - data = response.json() - response.raise_for_status() - except: - logging.error("Request failed for sentence '{}'".format(sentence)) - continue - - if "matches" in data: - # Number of issues - count = len(data["matches"]) - for match in data["matches"]: - # {'message': 'Possible spelling mistake found', 'shortMessage': 'Spelling mistake', 'replacements': [{'value': 'Woman'}, {'value': 'Roman'}, {'value': 'Domain'}, {'value': 'Oman'}] - if "message" in match: - suggestions = list(map(lambda x: x.popitem()[1], match["replacements"])) - print( - f"[{RED}FAIL{CEND}] {match['message']} in sentence '{sentence}'. Possible suggestions: {suggestions}" - ) - print(f"[{YELLOW}SUCCESS{CEND}] Grammar and spelling check complete") diff --git a/tools/help.py b/tools/help.py deleted file mode 100755 index 67c56d56a9..0000000000 --- a/tools/help.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -import yaml -from yaml.error import YAMLError -import os -import sys -from typing import Any, Optional -from enum import Enum - - -class ComponentType(Enum): - """ - Enum to allow for identifying the type of a plugin component (Connection, Action, Trigger) - """ - - connection = "connection" - action = "action" - trigger = "trigger" - - -class IO(object): - """ - An Input or Output on a Connection, Action, Trigger - """ - - def __init__( - self, - identifier: str, - title: Optional[str], - description: Optional[str], - type_: str, - required: bool, - default: Optional[Any], - enum: Optional[list] = None, - raw_parameters: set = {}, - ): - """ - Initializer for an input or output belonging to a component - :param identifier: Identifier for an IO (optional) - :param title: Title of an IO (optional) - :param description: Description of an IO (optional) - :param type_: Type of an IO (eg. string, integer, number, boolean, etc) - :param required: Boolean indicator of requiredness - :param default: Default value for the IO (optional) - :param enum: Enum values (optional) - :param raw_parameters: All properties belonging to the IO as a set of strings - - """ - self.identifier = identifier - self.title = title - self.description = description - self.type_ = type_ - self.required = False if not required else True - self.default = default - self.enum = enum - self.raw_parameters = raw_parameters - - def __lt__(self, other): - return self.identifier < other.identifier - - @property - def is_custom(self): - """ - Returns whether or not the IO is of a custom/non-standard type. - Standard types defined at Standard types defined at https://docs.rapid7.com/insightconnect/plugin-spec/#base-types - :return: Boolean indicating if a type is custom - """ - # Strip off possible list indicator and compare against standard types - return self.type_.lstrip("[]") not in { - "boolean", - "integer", - "int", - "number", - "float", - "string", - "date", - "bytes", - "object", - "password", - "python", - "file", - "credential_username_password", - "credential_asymmetric_key", - "credential_secret_key", - "credential_token", - } - - @classmethod - def from_dict(cls, raw: {str: Any}): - # Get the first and only key from the dict anonymously and assign that to an identifier - identifier = list(raw.keys())[0] - - return cls( - identifier=identifier, - title=raw[identifier].get("title"), - description=raw[identifier].get("description"), - type_=raw[identifier].get("type"), - required=raw[identifier].get("required"), - default=raw[identifier].get("default"), - enum=raw[identifier].get("enum"), - raw_parameters=set(raw[identifier].keys()), - ) - - -class CustomType(object): - """ - A CustomType object, comprised of a single identifier and multiple IO components - """ - - def __init__(self, identifier: str, properties: [IO]): - self.identifier = identifier - self.properties = properties - - -class PluginComponent(object): - """ - A Connection, Action, or Trigger - """ - - def __init__( - self, - component_type: ComponentType, - identifier: str = None, - title: Optional[str] = None, - description: Optional[str] = None, - inputs: [IO] = None, - outputs: [IO] = None, - raw_parameters: set = {}, - ): - """ - Initializer for a PluginComponent - :param component_type: Type of the component - :param title: Title of the component. Not present on a Connection component - :param description: Description of the component. Not present on a Connection component - :param inputs: List of component inputs (optional) - :param outputs: List of component outputs (never present on a Connection component) (optional) - :param raw_parameters: All top-level properties (input/output/etc) belonging to the component as a set of strings - """ - self.component_type = component_type - self.identifier = identifier - self.title = title - self.description = description - self.inputs = inputs - self.outputs = outputs - self.raw_parameters = raw_parameters - - def __lt__(self, other): - return self.identifier < other.identifier - - @classmethod - def new_action(cls, raw: dict): - # Get the first and only key from the dict anonymously and assign that to an identifier - identifier: str = list(raw.keys())[0] - raw_parameters: {str} = set(raw[identifier].keys()) - - input_, output = raw[identifier].get("input"), raw[identifier].get("output") - - inputs: [IO] = [IO.from_dict(raw={k: v}) for k, v in input_.items()] if input_ else [] - outputs: [IO] = [IO.from_dict(raw={k: v}) for k, v in output.items()] if output else [] - - return cls( - component_type=ComponentType.action, - identifier=identifier, - title=raw[identifier].get("title"), - description=raw[identifier].get("description"), - inputs=inputs, - outputs=outputs, - raw_parameters=raw_parameters, - ) - - @classmethod - def new_trigger(cls, raw: dict): - # Get the first and only key from the dict anonymously and assign that to an identifier - identifier: str = list(raw.keys())[0] - raw_parameters: {str} = set(raw[identifier].keys()) - - input_, output = raw[identifier].get("input"), raw[identifier].get("output") - - inputs: [IO] = [IO.from_dict(raw={k: v}) for k, v in input_.items()] if input_ else [] - outputs: [IO] = [IO.from_dict(raw={k: v}) for k, v in output.items()] if output else [] - - return cls( - component_type=ComponentType.trigger, - identifier=identifier, - title=raw[identifier].get("title"), - description=raw[identifier].get("description"), - inputs=inputs, - outputs=outputs, - raw_parameters=raw_parameters, - ) - - @classmethod - def new_connection(cls, raw: dict): - # Create a list of IO objects from the raw inputs dict - inputs: [IO] = [IO.from_dict(raw={k: v}) for k, v in raw.items()] if raw else [] - return cls(component_type=ComponentType.connection, identifier="connection", inputs=inputs) - - -class PluginSpec(object): - """ - A plugin specification file - """ - - def __init__( - self, - spec_version: str, - name: str, - title: str, - description: str, - version: str, - vendor: str, - tags: [str], - types: [CustomType], - connection: PluginComponent = None, - actions: [PluginComponent] = [], - triggers: [PluginComponent] = [], - ): - self.spec_version = spec_version - self.name = name - self.title = title - self.description = description - self.version = version - self.vendor = vendor - self.tags = tags - self.types = types - self.connection = connection - self.actions = actions - self.triggers = triggers - - @classmethod - def load_from_file(cls, path_): - if not os.path.isfile(path_): - return None - - f = open(path_, "r") - try: - spec = yaml.safe_load(f) - except YAMLError as e: - raise Exception("Error: Provided spec file was not valid YAML!") from e - f.close() - - custom_types: [CustomType] = [] - for identifier, properties in spec.get("types", {}).items(): - io = [IO.from_dict(raw={k: v}) for k, v in properties.items()] - custom_types.append(CustomType(identifier=identifier, properties=io)) - - connection = PluginComponent.new_connection(raw=spec.get("connection")) if spec.get("connection") else None - actions = ( - [PluginComponent.new_action(raw={k: v}) for k, v in spec.get("actions").items()] - if spec.get("actions") - else [] - ) - triggers = ( - [PluginComponent.new_action(raw={k: v}) for k, v in spec.get("triggers").items()] - if spec.get("triggers") - else [] - ) - - return cls( - spec_version=spec.get("plugin_spec_version"), - name=spec.get("name"), - title=spec.get("title"), - description=spec.get("description"), - version=spec.get("version"), - vendor=spec.get("vendor"), - tags=spec.get("tags"), - types=custom_types, - connection=connection, - actions=actions, - triggers=triggers, - ) - - -class Help(object): - """ - Class for providing yaml-to-markdown (spec to help file) functionality - """ - - # Represents a linebreak in Markdown - MD_BREAK = "\n\n" - - def __init__(self, spec: PluginSpec): - """ - Initialize a Help object - :param spec: PluginSpec object to load - """ - self.spec = spec - - self.actions: [str] = [] - self.triggers: [str] = [] - self.connection: str = None - - @classmethod - def load_from_file(cls, path_: str): - """ - Convenience initializer for a Help object - :param path_: Path to a plugin.spec.yaml - :return: New Help object - """ - spec = PluginSpec.load_from_file(path_=path_) - return cls(spec=spec) - - @staticmethod - def _generate_input_table(inputs: [IO]) -> str: - """ - Generates a markdown table for component inputs - :param inputs: List of IO objects for a component - :return: Markdown table as a string - """ - table = "|Name|Type|Default|Required|Description|Enum|\n" "|----|----|-------|--------|-----------|----|\n" - - for counter, io in enumerate(sorted(inputs), 1): - # if io.is_custom: - # # Custom type, so link to the custom type table. - # # Link could be for an array of custom types, so strip the array symbol off otherwise a broken link - # # will be generated. - # type_text = f"[{io.type_}](#{io.type_.lstrip('[]')})" - # else: - # type_text = io.type_ - - table += f"|{io.identifier}|{io.type_}|{io.default}|{io.required}|{io.description}|{io.enum}|" - table += "\n" if counter < len(inputs) else "" - - return table - - @staticmethod - def _generate_output_table(outputs: [IO]) -> str: - """ - Generates a markdown table for component outputs - :param outputs: List of IO objects for a component - :return: Markdown table as a string - """ - table = "|Name|Type|Required|Description|\n" "|----|----|--------|-----------|\n" - - for counter, io in enumerate(sorted(outputs), 1): - # if io.is_custom: - # # Custom type, so link to the custom type table. - # # Link could be for an array of custom types, so strip the array symbol off otherwise a broken link - # # will be generated. - # type_text = f"[{io.type_}](#{io.type_.lstrip('[]')})" - # else: - # type_text = io.type_ - - table += f"|{io.identifier}|" f"{io.type_}|" f"{io.required}|" f"{io.description}|" - table += "\n" if counter < len(outputs) else "" - - return table - - def _generate_component_section(self, section_type: ComponentType) -> str: - """ - Generates a markdown help section for a class of components (actions or triggers) - :param section_type: ComponentType enum value representing the section type to generate - :return: Markdown section as a string - """ - assert ( # noqa: B101 - section_type != ComponentType.connection - ), "generate_component_section does not support connections!" - - # Both actions and triggers are of type PluginComponent - so they can be swapped in/out in the code below - components = self.spec.actions if section_type == ComponentType.action else self.spec.triggers - singular = "action" if section_type == ComponentType.action else "trigger" - - markdown = f"## {singular.capitalize()}s{self.MD_BREAK}" - if not components: - markdown += f"_This plugin does not contain any {singular}s._{self.MD_BREAK}" - return markdown - - for component in sorted(components): - # Base: Based on the first word of the description, determine the help description base sentence - # Description: Lowercase the first letter of the first word so it flows w/ the base sentence - has_s = component.description.split(" ")[0].endswith("s") - base = f"This {singular}" if has_s else f"This {singular} is used to" - description = component.description[0].lower() + component.description[1:] - - markdown += f"### {component.title}{self.MD_BREAK}" f"{base} {description}.{self.MD_BREAK}" - if component.inputs: - markdown += ( - f"#### Input{self.MD_BREAK}" f"{self._generate_input_table(inputs=component.inputs)}{self.MD_BREAK}" - ) - if component.outputs: - markdown += ( - f"#### Output{self.MD_BREAK}" - f"{self._generate_output_table(outputs=component.outputs)}{self.MD_BREAK}" - f"Example output:{self.MD_BREAK}" - f"```\n" - f"```{self.MD_BREAK}" - ) - - return markdown - - def _generate_about_section(self) -> str: - """ - Generates a markdown About section - :return: Markdown string as section - """ - - try: - leads_with_title = self.spec.description.index(self.spec.title) == 0 - except ValueError: # Title not found at all in the description - leads_with_title = False - - description = self.spec.description.replace(self.spec.title, "", 1 if leads_with_title else 0) - markdown = ( - f"## About{self.MD_BREAK}" - f"[{self.spec.title}](LINK TO PRODUCT/VENDOR WEBSITE) {description}." - f"{self.MD_BREAK}" - ) - - return markdown - - def generate_full_markdown(self) -> str: - """ - Generates a formatted plugin help file as a markdown string - :return: Markdown help file string - """ - markdown = "" - - # Title - markdown += f"# {self.spec.title}{self.MD_BREAK}" - - # About section - markdown += self._generate_about_section() - - # Actions - markdown += self._generate_component_section(section_type=ComponentType.action) - - # Triggers - markdown += self._generate_component_section(section_type=ComponentType.trigger) - - # Connection - markdown += f"## Connection{self.MD_BREAK}" - if self.spec.connection: - markdown += ( - f"The connection configuration accepts the following parameters:{self.MD_BREAK}" - f"{self._generate_input_table(inputs=self.spec.connection.inputs)}{self.MD_BREAK}" - ) - else: - markdown += f"_This plugin does not contain a connection._{self.MD_BREAK}" - - # Troubleshooting - markdown += ( - f"## Troubleshooting{self.MD_BREAK}" - f"_This plugin does not contain any troubleshooting information._{self.MD_BREAK}" - ) - - # Workflows - markdown += f"## Workflows{self.MD_BREAK}" f"Examples:{self.MD_BREAK}" f"* EXAMPLE HERE {self.MD_BREAK}" - - # Versions - markdown += f"## Versions{self.MD_BREAK}" f"* 1.0.0 - Initial plugin{self.MD_BREAK}" - - # References - markdown += ( - f"## References{self.MD_BREAK}" f"* [{self.spec.title}](LINK TO PRODUCT/VENDOR WEBSITE){self.MD_BREAK}" - ) - - # Custom Types - markdown += f"## Custom Output Types{self.MD_BREAK}" - if self.spec.types: - for type_ in self.spec.types: - markdown += ( - f"### {type_.identifier}{self.MD_BREAK}" - f"{self._generate_output_table(outputs=type_.properties)}{self.MD_BREAK}" - ) - else: - markdown += f"_This plugin does not contain any custom output types._" - - return markdown - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("./help.py ") - sys.exit(0) - - path = sys.argv[1] - - help_ = Help.load_from_file(path_=path) - help_markdown = help_.generate_full_markdown() - print(help_markdown) diff --git a/tools/help_conversion.py b/tools/help_conversion.py deleted file mode 100644 index 13a142603e..0000000000 --- a/tools/help_conversion.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -from help_update import HelpUpdate - - -def main(): - subdirs = list(filter(lambda d: os.path.isdir(d), os.listdir("."))) - base_path = os.getcwd() - - for d in subdirs: - os.chdir(os.path.join(base_path, d)) - try: - output = HelpUpdate.convert() - except Exception: # noqa: B112 - continue - if output is not None: - with open("./help.md", "w") as h: - print("Writing for " + d) - h.write(output) - - -if __name__ == "__main__": - main() diff --git a/tools/help_update.py b/tools/help_update.py deleted file mode 100755 index 466dcd48fd..0000000000 --- a/tools/help_update.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -import re -import os - - -class HelpUpdate: - @staticmethod - def convert(): - help_file = "./help.md" - try: - with open(help_file) as h: - contents = h.read() - except FileNotFoundError: - return None - description = HelpUpdate.get_description(contents) - key_features = HelpUpdate.get_key_features() - requirements = HelpUpdate.get_requirements() - documentation = HelpUpdate.get_documentation(contents) - versions = HelpUpdate.get_versions(contents) - links = HelpUpdate.get_links(contents) - - output = description + key_features + requirements + documentation + versions + links - return output - - @staticmethod - def get_description(contents: str) -> str: - pattern = "## About([\s\S]*?)##" - match: str = re.findall(pattern, contents)[0] - return "# Description\n\n" + match.strip() + "\n\n" - - @staticmethod - def get_key_features() -> str: - key_features = """# Key Features\n -* Feature 1 -* Feature 2 -* Feature 3 -""" - return key_features + "\n" - - @staticmethod - def get_requirements() -> str: - requirements = """# Requirements\n -* Example: Requires an API Key from the product -* Example: API must be enabled on the Settings page in the product -""" - return requirements + "\n" - - @staticmethod - def get_documentation(contents: str) -> str: - output = "# Documentation\n\n" - - # grab connection info - pattern = "## Connection([\s\S]*?)## " - match: str = re.findall(pattern, contents)[0] - setup = "## Setup\n\n" + match.strip() + "\n\n" - - # grab details - pattern = "## Actions([\s\S]*?)## Triggers" - match = re.findall(pattern, contents)[0] - match = re.sub("# ", "## ", match) - details = "## Technical Details\n\n" + "### Actions\n\n" + match.strip() + "\n\n" - - # grab triggers - pattern = "## Triggers([\s\S]*?)## Connection" - match = re.findall(pattern, contents)[0] - match = re.sub("# ", "## ", match) - details += "### Triggers\n\n" + match.strip() + "\n\n" - - # grab troubleshooting - pattern = "## Troubleshooting([\s\S]*?)## " - match = re.findall(pattern, contents)[0] - troubleshooting = "## Troubleshooting\n\n" + match.strip() + "\n\n" - - # grab custom types - pattern = "## Custom Output Types([\s\S]*)## Workflows" - try: - match = re.findall(pattern, contents)[0] - match = re.sub("# ", "## ", match) - except IndexError: - match = "_This plugin does not contain any custom output types._" - types = "### Custom Output Types\n\n" + match.strip() + "\n\n" - - output += setup + details + types + troubleshooting - return output - - @staticmethod - def get_versions(contents: str) -> str: - pattern = "## Versions([\s\S]*?)## " - match = re.findall(pattern, contents)[0].strip() - lines: list[str] = match.split("*") - lines.reverse() - lines.remove("") - if not lines[0].endswith("\n"): - lines[0] += "\n" - output = "".join(["*" + line for line in lines]) - return "# Version History\n\n" + output + "\n" - - @staticmethod - def get_links(contents: str) -> str: - output = "# Links\n\n" - - try: - pattern = "## References([\s\S]*?)## " - match = re.findall(pattern, contents)[0] - except IndexError: - pattern = "## References([\s\S]*)" - match = re.findall(pattern, contents)[0] - references = "## References\n\n" + match.strip() + "\n\n" - - output += references - return output diff --git a/tools/json_to_spec.py b/tools/json_to_spec.py deleted file mode 100755 index ce6f09d734..0000000000 --- a/tools/json_to_spec.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -import json -from past.builtins import basestring -from future.utils import iteritems -import sys -from collections import OrderedDict -from jinja2 import Template -import os - - -# set to true if you don't want to try to generate nested types -OBJECTS_ONLY = False - -filename = sys.argv[1] - -this_dir = os.path.dirname(__file__) -rel_path = "templates/template.json_to_spec.j2" -abs_path_template = os.path.join(this_dir, rel_path) - -with open(filename) as data_file: - data = json.load(data_file) - -with open(abs_path_template) as f: - spec_template = Template(f.read()) - - -type_refs = {} -types = OrderedDict() -output = OrderedDict() - - -def type_name_unused(name): - if not name in types: - return name - i = 0 - while True: - name = "%s_%d" % (name, i) - if not name in types: - return name - i += 1 - - -def detect_type(name, value, level=0): - if isinstance(value, basestring): - return "string" - if isinstance(value, bool): - return "boolean" - if isinstance(value, int): - return "integer" - if isinstance(value, float): - return "float" - if isinstance(value, list): - if value: - return '"[]{}"'.format(detect_type(name, value[0], level + 1)) - else: - return "[]object" - - if isinstance(value, dict): - keys = sorted(value.keys()) - if len(keys) == 0 or OBJECTS_ONLY: - return "object" - list_key = str(keys) - if list_key in type_refs: - return type_refs[list_key] - - type_name = type_name_unused(name) - typ = {} - for k in keys: - typ[str(k)] = { - "title": str(k).replace("_", " ").title(), - "type": detect_type(k, value[k]), - "description": str(k).replace("_", " ").capitalize(), - } - type_refs[list_key] = type_name - types[str(type_name)] = typ - return str(type_name) - - -def dump_to_spec(sections): - return spec_template.render(sections=sections) - - -def build_spec(item): - if not isinstance(item, dict): - raise "Not a object. You must start with a JSON object, not an array or any other type: %s" % item - for key, value in iteritems(item): - output[str(key)] = { - "title": str(key).replace("_", " ").title(), - "type": detect_type(key, value), - "description": str(key).replace("_", " ").capitalize(), - } - typesection = {"types": types} - outputsection = {"output": output} - sections = {} - sections.update(typesection) - sections.update(outputsection) - - print(dump_to_spec(sections)) - - -build_spec(data) diff --git a/tools/mdl.sh b/tools/mdl.sh deleted file mode 100755 index 2da1ad67c8..0000000000 --- a/tools/mdl.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -BOLD=$(tput bold) -UNDERLINE=$(tput smul) -NORMAL=$(tput sgr0) -RED=$(tput setaf 1) -YELLOW=$(tput setaf 3) - - -is_help() { - if [[ -f "help.md" ]]; then - return 0 - fi - printf "\n[${YELLOW}*${NORMAL}] Plugin is missing help.md, to generate it run: make help\n" && return 1 -} - -is_mdl() { - if [[ -x $(which mdl || echo "None") ]]; then - return 0 - fi - printf "\n[${YELLOW}*${NORMAL}] ${UNDERLINE}For Markdown linting, install mdl:${NORMAL} gem install mdl\n" && return 1 -} - -main() { - is_mdl || exit 0 - is_help || exit 0 - printf "\n[${YELLOW}*${NORMAL}] ${BOLD}Validating markdown...${NORMAL}\n" - mdl --rules ~MD024,~MD013,~MD029,~MD033,~MD034,~MD013,~MD024,~MD025,~MD034,~MD036 help.md - result=$? - if [[ $result > 0 ]]; then - printf "[${RED}FAIL${NORMAL}] Fails markdown linting\n" - else - printf "[${YELLOW}SUCCESS${NORMAL}] Passes markdown linting\n" - fi - - return $result -} - -main diff --git a/tools/misspell.sh b/tools/misspell.sh deleted file mode 100755 index a5c9362bf9..0000000000 --- a/tools/misspell.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -BOLD=$(tput bold) -UNDERLINE=$(tput smul) -NORMAL=$(tput sgr0) -RED=$(tput setaf 1) -YELLOW=$(tput setaf 3) - - -is_misspell() { - if [[ -x $(command -v misspell || echo "None") ]]; then - return 0 - fi - printf "\n[${YELLOW}*${NORMAL}] ${UNDERLINE}For spell checking, install misspell:${NORMAL} go get -u github.com/client9/misspell/cmd/misspell\n" && return 1 -} - -main () { - is_misspell || exit 0 - printf "\n[${YELLOW}*${NORMAL}] ${BOLD}Checking for typos with Misspell...${NORMAL}\n" - output=$(misspell .) - if [[ "$output" ]]; then - printf "[${RED}FAIL${NORMAL}] Fails Misspell check\n" - echo $output - else - printf "[${YELLOW}SUCCESS${NORMAL}] Passes Misspell check\n" - fi - return 0 -} - -main \ No newline at end of file diff --git a/tools/pyflakes.sh b/tools/pyflakes.sh deleted file mode 100755 index 48207a3546..0000000000 --- a/tools/pyflakes.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -BOLD=$(tput bold) -UNDERLINE=$(tput smul) -NORMAL=$(tput sgr0) -RED=$(tput setaf 1) -YELLOW=$(tput setaf 3) - - -is_python() { - if [[ -f "setup.py" ]]; then - return 0 - fi - return 1 -} - -is_pyflakes() { - if [[ -x $(which pyflakes || echo "None") ]]; then - return 0 - fi - printf "\n${UNDERLINE}For Python linting, install pyflakes:${NORMAL} pip install --upgrade pyflakes\n" && return 1 -} - - -main() { - is_python || exit 0 - is_pyflakes || exit 0 - printf "\n[${YELLOW}*${NORMAL}] ${BOLD}Validating python files...${NORMAL}\n" - pyflakes $(find . -type f -name "*.py" | egrep -v '__init__.py|schema.py') - result=$? - [[ $result -eq 0 ]] && printf "[${YELLOW}SUCCESS${NORMAL}] Passes pyflakes linting\n" || printf "[${RED}FAIL${NORMAL}] Fails pyflakes linting\n" - return 0 # Make make happy :) -} - -main diff --git a/tools/run.sh b/tools/run.sh deleted file mode 100755 index 054b50ce07..0000000000 --- a/tools/run.sh +++ /dev/null @@ -1,388 +0,0 @@ -#!/bin/bash -# Initializations -END="$(tput sgr0)" -UNDERLINE="$(tput smul)" -YELLOW="$(tput setaf 3)" -RED="$(tput setaf 1)" -BLUE="$(tput setaf 4)" ORANGE=$(tput setaf 172) -MAGENTA="$(tput setaf 5)" -CYAN="$(tput setaf 6)" -WHITE="$(tput setaf 7)" -ARGC=$# -SPEC="./plugin.spec.yaml" -ORDER_FILE="tests/order.txt" -DEBUG_TRUE=0 -VOL_TRUE=0 -CACHE_TRUE=0 -PORT_TRUE=0 -MAKE_TRUE=0 -JSON_RUN=0 -JSON_TEST_ACK=0 -JSON_TEST_ALL=0 -JSON_RUN_ALL=0 -JSON_RUN_ACK=0 -ASSESS=0 -COMMANDS='info|sample|samples|test|run|http|bash|stats' -ARGV="$@" -CMD="$ARGV" -EARG=$3 # Extra arg for sample name and stats options - -# functions -usage() -{ -cat < | all) all - Execution order can be specified via tests/order.txt - ${YELLOW}-R${END} Run JSON run methods (e.g. | all) all - Execution order can be specified via tests/order.txt -${WHITE}Usage${END}: $0 [-c ] [-j] [-d] [-A] [-C] [-m] [-T ] [-R ] -v -p -EOF -} - -die(){ - printf -- "${RED}$*${END}\n" >&2 - exit 1 -} - -hi(){ - printf -- "${YELLOW}$*${END}\n" >&2 -} - -info(){ - printf -- "${MAGENTA}$*${END}\n" >&2 -} - -get_vendor(){ - local vendor - vendor=$(grep '^vendor: ' $SPEC | sed 's/vendor: //') - printf "$vendor" -} - -get_name(){ - local name - name=$(grep '^name: ' $SPEC | sed 's/name: //') - printf "$name" -} - -get_version(){ - local version - version=$(grep '^version: ' $SPEC | sed 's/version: //') - printf "$version" -} - -get_plugin_spec_version(){ - local version - version=$(grep '^plugin_spec_version: ' $SPEC | sed 's/plugin_spec_version: //') - printf "$version" -} - -get_actions(){ - local actions - actions=$(../tools//stats.py $SPEC | grep 'Action List:' | sed 's/ Action List: //') - printf "$actions" -} - -get_triggers(){ - local triggers - triggers=$(../tools/stats.py $SPEC | grep 'Trigger List:' | sed 's/ Trigger List: //') - printf "$triggers" -} - -get_stats(){ - local stats - local arg=$1 - [[ $arg ]] && cd .. && stats=$(./tools/stats.py $arg) && cd $OLDPWD || stats=$(../tools/stats.py $SPEC) - printf "$stats" -} - -files_by_order(){ - local files - if [[ -f $ORDER_FILE ]]; then - [[ -s $ORDER_FILE ]] || die "Order file is empty! tests/order.txt requires a list of JSON test filenames to guarantee the execution order when using the all argument." - files=$(cat $ORDER_FILE) - else - files=tests/* - fi - printf "$files" -} - -run_json_method(){ - local method="$1" - [[ $ASSESS = 1 ]] && JQ="| grep -- ^\{ | jq -r '.body | try(.log | split(\"\\\\n\") | .[]),.output'" && create_assessment_header "$method" - - # JSON test file logic - [[ $method == "test" ]] && [[ $JSON_TEST_ALL = 1 ]] && FILES=$(files_by_order) - [[ $method == "run" ]] && [[ $JSON_RUN_ALL = 1 ]] && FILES=$(files_by_order) - - - [[ $method == "test" ]] && FILE=$TEST_FILE - [[ $method == "run" ]] && FILE=$RUN_FILE - - for json in $FILE $FILES; do - [[ -f $json ]] || die "File $json does not exist, can't continue" - grep -q '"trigger":' $json && DEBUG="--debug" && TRIGGER=1 - RUN="cat $json | docker run --rm "$VOLUME" "$PORT_FWD" -i "${VENDOR}/${NAME}:${VERSION}" "$DEBUG" "$method" ${JQ}" - hi "Running: $RUN" - [[ $TRIGGER = 1 ]] && info "This is a trigger ^C to continue" && unset TRIGGER - [[ $ASSESS = 1 ]] && json=$(eval $RUN) || eval $RUN - [[ $DEBUG_TRUE = 0 ]] && unset DEBUG - [[ $ASSESS = 1 ]] && create_assessment_output "$json" - printf "\n" - done - unset FILE - unset FILES -} - -create_assessment_header(){ - header="$(tr '[:lower:]' '[:upper:]' <<< ${1:0:1})${1:1}" - printf -- "### $header\n" - printf -- "Autogenerated with:\n\`$0 $ARGV\`\n" -} - -create_assessment_validation(){ - printf -- '```\n' - printf -- "$(make validate)\n" - printf -- '```\n' - printf -- '\n' -} - -create_assessment_output(){ - local json="$1" - printf -- "
\n\n" - printf -- '```\n' - printf -- '%s\n' "${json}" - printf -- '```\n' - printf -- "\n" - printf -- "\$ ${RUN}\n" - printf -- "\n" - printf -- "
\n" -} - -create_assessment_end(){ - printf "### UI\n\nScreenshots of the plugin being used in Komand for validating use go below this line.\n" - printf "#### Workflow Builder\n\n" - printf "#### Job\n\n" - printf "#### Artifact\n\n" -} - -# if less than 1 argument -if [[ $ARGC < 1 ]]; then - usage - exit 1 -fi - -while getopts "hAc:Cdf:jmT:R:f:p:v:" OPTION; do - case $OPTION in - v) - VOL_TRUE=1 - if [[ -d "${OPTARG%:*}" ]]; then - VOL="$OPTARG" - else - die "Directory $VOL doesn't exist" - fi - [[ $VOL =~ : ]] || die "Invalid argument as e.g. \`\`-v /var/cache:/var/cache\'\'" ;; - p) - PORT_TRUE=1 - PORTS="$OPTARG" - [[ $PORTS =~ [0-9]+:[0-9]+ ]] || die "Invalid argument as e.g. \`\`-p 8000:8888\'\'" - ;; - c) - CMD="$OPTARG" - if [[ "$CMD" == info ]]; then - : - elif [[ "$CMD" == sample ]]; then - : - elif [[ "$CMD" == samples ]]; then - : - elif [[ "$CMD" == test ]]; then - : - elif [[ "$CMD" == run ]]; then - : - elif [[ "$CMD" == http ]]; then - : - elif [[ "$CMD" == bash ]]; then - : - elif [[ "$CMD" == stats ]]; then - : - else - die "Unknown command type! ($COMMANDS)" - fi - ;; - C) - CACHE_TRUE=1;; - T) - if [[ -f "$OPTARG" ]]; then - TEST_FILE="$OPTARG" - JSON_TEST_ACK=1 - elif [[ "$OPTARG" == all ]]; then - JSON_TEST_ALL=1 - JSON_TEST_ACK=1 - else - die "Input JSON file $OPTARG doesn't exist" - fi - ;; - R) - if [[ -f "$OPTARG" ]]; then - RUN_FILE="$OPTARG" - JSON_RUN_ACK=1 - elif [[ "$OPTARG" == all ]]; then - JSON_RUN_ALL=1 - JSON_RUN_ACK=1 - else - die "Input JSON file $OPTARG doesn't exist" - fi - ;; - A) - ASSESS=1;; - f) - if [[ -f "$OPTARG" ]]; then - SPEC="$OPTARG" - else - die "Input plugin spec file $OPTARG doesn't exist" - fi - ;; - j) - JQ="| grep -- ^\{ | jq -r '.body | try(.log | split(\"\\\\n\") | .[]),.output'";; - m) - MAKE_TRUE=1;; - d) - DEBUG_TRUE=1 - DEBUG="--debug";; - h) - usage - exit 1 - ;; - \?) - exit 1 - ;; - esac -done - -# Volume -if [[ $VOL_TRUE = 1 ]]; then - VOLUME="$VOL" -fi - -# Cache -if [[ $CACHE_TRUE = 1 ]]; then - VOLUME="${VOL:-/var/cache:/var/cache}" -fi - -if [[ $CACHE_TRUE = 1 ]] || [[ $VOL_TRUE = 1 ]]; then - VOLUME="-v $VOLUME" -fi - -# Port -if [[ $PORT_TRUE = 1 ]]; then - PORT_FWD="-p $PORTS" -fi - -VENDOR=$(get_vendor) -NAME=$(get_name) -VERSION=$(get_version) -PLUGIN_SPEC_VERSION=$(get_plugin_spec_version) - -# Stats -if [[ $CMD = "stats" ]]; then - if [[ "$EARG" == "all" ]]; then - EARG=total - elif [[ "$EARG" == "python" ]]; then - : - elif [[ "$EARG" == "go" ]]; then - : - elif [[ "$EARG" == "" ]]; then - : - else - die "Stats requires an argument e.g. \`\`./run.sh -c stats [all|python|go]\'\'" - fi - get_stats $EARG - exit 0 -fi - -# Make -if [[ $MAKE_TRUE = 1 ]]; then - hi "Building plugin first" - [[ $SPEC =~ '..' ]] && BUILD="../$NAME" && make -C $BUILD || make -fi - -if [[ $CMD == "samples" ]]; then - [[ -d tests ]] || mkdir "tests" - TRIGGERS=$(get_triggers) - ACTIONS=$(get_actions) - for sample in $TRIGGERS $ACTIONS; do - RUN="docker run --rm -ti "${VENDOR}/${NAME}:${VERSION}" sample $sample | jq '.' > tests/${sample}.json" - hi "Running: $RUN" - eval $RUN - done - exit 0 -fi - -if [[ $CMD == "bash" ]]; then - [[ $EARG ]] && CMD="$EARG" - RUN="docker run --rm --entrypoint "$CMD" "$VOLUME" "$PORT_FWD" -ti "${VENDOR}/${NAME}:${VERSION}" "$DEBUG" ${JQ}" - hi "Running: $RUN" - eval $RUN - exit 0 -fi - -[[ $ASSESS = 1 ]] && printf "## Assessment\n" -[[ $ASSESS = 1 ]] && create_assessment_header Validation "make validate" && create_assessment_validation -[[ $JSON_TEST_ACK == 1 ]] && run_json_method "test" -[[ $JSON_RUN_ACK == 1 ]] && run_json_method "run" -[[ $ASSESS = 1 ]] && create_assessment_end - -# Exit if we ran run_json_method as everything after are plugin bin commands -[[ $JSON_RUN_ACK == 1 ]] && exit 0 -[[ $JSON_TEST_ACK == 1 ]] && exit 0 - -# Start plugin bin commands -if [[ "$CMD" == info ]]; then - : -elif [[ "$CMD" == sample ]]; then - if [[ $EARG ]]; then - CMD="$CMD $EARG | jq '.'" - else - hi "Actions: [$(get_actions)]" - hi "Triggers: [$(get_triggers)]" - die "Sample requires sample name e.g. \`\`./run.sh -c sample \'\'" - fi -elif [[ "$CMD" == test ]]; then - : -elif [[ "$CMD" == run ]]; then - : -elif [[ "$CMD" == http ]]; then - - if [[ $PLUGIN_SPEC_VERSION != v2 ]]; then - die "Plugin does not support proxy mode" - fi - - # 10001 is the default start port for the plugin proxy, so we repeat its use here - i=10001 - - # Loop until there's an open port - until ! nc -z localhost ${i} &>/dev/null - do - i=$((i + 1)) - done - printf "${YELLOW}Forwarding to port ${i}${END}\n" - - PORT_FWD="-d $PORT_FWD -p ${i}:10001" -else - unset CMD -fi - -RUN="$JSON_TEST docker run --rm "$VOLUME" "$PORT_FWD" -i "${VENDOR}/${NAME}:${VERSION}" "$DEBUG" "$CMD" ${JQ}" -hi "Running: $RUN" -eval $RUN - -exit 0 diff --git a/tools/spec_resources_update.py b/tools/spec_resources_update.py deleted file mode 100644 index 65178ad9a2..0000000000 --- a/tools/spec_resources_update.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import re - - -def main(): - subdirs = list(filter(lambda d: os.path.isdir(d), os.listdir("."))) - base_path = os.getcwd() - - for d in subdirs: - spec_path = os.path.join(base_path, d, "plugin.spec.yaml") - try: - output = update_source_url(spec_path, d) - except Exception: # noqa: B112 - continue - if output is not None: - with open(spec_path, "w") as h: - print("Writing for " + d) - h.write(output) - - -def update_source_url(spec_path, plugin_name): - source_url = f"https://github.com/rapid7/insightconnect-plugins/tree/master/{plugin_name}" - license_url = "https://github.com/rapid7/insightconnect-plugins/blob/master/LICENSE" - new_resources = f"""resources: - source_url: {source_url} - license_url: {license_url} -""" - pattern = "(resources:[\s\S]*?)tags:" - with open(spec_path, "r") as spec: - content = spec.read() - match = re.findall(pattern, content)[0] - content = content.replace(match, new_resources, 1) - return content - - -if __name__ == "__main__": - main() diff --git a/tools/spec_tag_update.py b/tools/spec_tag_update.py deleted file mode 100644 index 3532fa9c05..0000000000 --- a/tools/spec_tag_update.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import re - - -def main(): - subdirs = list(filter(lambda d: os.path.isdir(d), os.listdir("."))) - base_path = os.getcwd() - - for d in subdirs: - spec_path = os.path.join(base_path, d, "plugin.spec.yaml") - try: - output = update_tags(spec_path) - except Exception: # noqa: B112 - continue - if output is not None: - with open(spec_path, "w") as h: - print("Writing for " + d) - h.write(output) - - -def update_tags(spec_path): - pattern = "tags:\n(?:-[^-^\n]*\n)*" - with open(spec_path, "r") as spec: - content = spec.read() - match = re.findall(pattern, content)[0] - tags = ", ".join([tag.strip() for tag in match.split("-")[1:]]) - addition = f"""hub_tags: - use_cases: [] - keywords: [{tags}] - features: [] -""" - content = content.replace(match, match + addition, 1) - return content - - -if __name__ == "__main__": - main() diff --git a/tools/spec_to_json.py b/tools/spec_to_json.py deleted file mode 100644 index 1e9f5eda40..0000000000 --- a/tools/spec_to_json.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python -import json -import os - - -SPECS_DIR = "_json_specs" - -os.chdir("..") -plugins_root = os.getcwd() -plugin_directories = os.listdir() # Essentially a list of plugin names -os.mkdir(SPECS_DIR) - -for plugin_directory in plugin_directories: - print("Parsing %s" % plugin_directory) - spec = os.path.join(plugins_root, plugin_directory, "plugin.spec.yaml") - if os.path.isfile(spec): - with open(spec, "r") as spec_file: - spec_contents = spec_file.read() - output = {"spec": spec_contents} - - os.chdir(os.path.join(plugins_root, SPECS_DIR)) - with open(plugin_directory + ".json", "w") as outfile: - json.dump(output, outfile) diff --git a/tools/spec_to_manifest.py b/tools/spec_to_manifest.py deleted file mode 100644 index 70a1c98d1c..0000000000 --- a/tools/spec_to_manifest.py +++ /dev/null @@ -1,234 +0,0 @@ -import json -import re -import sys -import yaml -import logging - - -def main(): - filename = "" - path = "" - correct_params = False - try: - filename = sys.argv[1] - path = sys.argv[2] - correct_params = True - except IndexError: - logging.warning("ERROR: Insufficient parameters") - - if correct_params: - if verify_yaml(filename): - spec = open_and_read_spec(filename) - print("Read spec file at " + filename) - json_obj = parse_fields(spec) - print("Converted fields") - write_to_json(json_obj, path) - print("Wrote manifest at " + path) - else: - logging.warning("ERROR: Wrong file type specified") - - -def parse_fields(yaml_obj): # noqa: MC0001 - # Manifest Version -- mandatory - manifest_obj = {"manifest_version": 1} - - resources = yaml_obj.get("resources") - cloud_ready = yaml_obj.get("cloud_ready") - unique_name = yaml_obj.get("name") - publisher = yaml_obj.get("vendor") - support = yaml_obj.get("support") - - # Extension -- mandatory - # From yaml, other info changes based on extension - extension_type = yaml_obj.get("extension") - extension_obj = {"type": extension_type} - if extension_type == "workflow": - extension_obj[extension_type] = unique_name + ".icon" - if extension_type == "plugin": - extension_obj["externalPluginName"] = unique_name - if cloud_ready: - extension_obj["cloudReady"] = "true" - else: - extension_obj["cloudReady"] = "false" - - manifest_obj["extension"] = extension_obj - - # ID -- mandatory - # From yaml - manifest_obj["id"] = unique_name - - # Title -- mandatory - # From yaml - manifest_obj["title"] = yaml_obj.get("title") - - # Overview -- Mandatory - # from yaml - manifest_obj["overview"] = yaml_obj.get("description") - - # Description -- mandatory - # Tricky extraction of description from .md - manifest_obj["description"] = "" - - # Key Features -- mandatory - # From .md - manifest_obj["key_features"] = [] - - # Requirements -- Optional - # Tricky reading from .md, due to different formatting.... - manifest_obj["requirements"] = [] - - # Resources -- optional - # Pull from .md - if resources: - vendor_url = resources.get("vendor_url") - if vendor_url: - manifest_obj["resources"] = [{"text": "Vendor Website", "url": vendor_url}] - else: - # Append references --> links from .md - manifest_obj["resources"] = [] - - # Version -- optional - # Directly from yaml, parsed as string in manifest - manifest_obj["version"] = str(yaml_obj.get("version")) - - # Version History.... - # Should be from .md -- need to parse - manifest_obj["version_history"] = [] - - # Publisher -- mandatory - # "Vendor" field from yaml - manifest_obj["publisher"] = publisher - - # Support -- mandatory - # from yaml if R7, where to get info on "community" support? - if support == "rapid7": - support_obj = {"type": "publisher", "url": "http://www.rapid7.com"} - else: - if vendor_url: - support_obj = {"type": "community", "url": vendor_url} - else: - support_obj = {"type": "community", "url": ""} - manifest_obj["support"] = support_obj - - # Rapid7 Products -- mandatory - # Products key from yaml, we don't have an understanding of primary/secondary.... - product_listings = yaml_obj.get("products") - r7_products = [] - for x in range(len(product_listings)): - if x == 0: - r7_products.append({"name": product_listings[x], "role": "primary"}) - else: - r7_products.append({"name": product_listings[x], "role": "secondary"}) - manifest_obj["rapid7_products"] = r7_products - - # Required R7 Features -- Required - # Different rules for workflows, plugins, etc... - - if extension_type == "workflow" or extension_type == "plugin": - if unique_name == "rapid7_insight_agent": - manifest_obj["required_rapid7_features"] = ["orchestrator", "agent"] - else: - if not cloud_ready: - manifest_obj["required_rapid7_features"] = ["orchestrator"] - else: - manifest_obj["required_rapid7_features"] = [] - - # Status -- optional - # From yaml - manifest_obj["status"] = yaml_obj.get("status") - - # Logos -- optional - manifest_obj["logos"] = {"primary": "extension.png", "secondary": []} - - # Display Options -- mandatory - # No sense of this in spec/md -- only "credit_author" is valid - manifest_obj["display_options"] = [] - - # Tags -- Mandatory - # Converting from yaml - hub_tags = yaml_obj.get("hub_tags") - - # Tags.categories -- Mandatory - categories = hub_tags.get("use_cases") - - # Tags.third_party_products -- optional - # This info is not readily available yet - third_party_products = [] - - # Tags.keywords -- optional - keywords = hub_tags.get("keywords") - manifest_obj["tags"] = { - "categories": categories, - "third_party_products": third_party_products, - "keywords": keywords, - } - - # Documentation -- optional - documentation_obj = {} - if resources.get("docs_url"): - documentation_obj["type"] = "url" - documentation_obj["source"] = resources.get("docs_url") - else: - documentation_obj["type"] = "file" - documentation_obj["source"] = "help.md" - - manifest_obj["documentation"] = documentation_obj - - # Media -- optional - # Conversions from yaml -- plugins don't have screenshots - if resources: - screenshots = resources.get("screenshots") - if screenshots: - media_list = [] - for media in yaml_obj.get("resources").get("screenshots"): - image_pattern = r"\S+\.(?:png|jpe?g)" - video_pattern = r"\S+\.(?:mp4)" - type_of_media = "" - if re.match(image_pattern, media["name"]): - type_of_media = "image" - if re.match(video_pattern, media["name"]): - type_of_media = "video" - media_info = { - "source": media.get("name"), - "title": media.get("title"), - "type": type_of_media, - } - media_list.append(media_info) - manifest_obj["media"] = media_list - - # Links -- optional - # Conversion from yaml - src = "" - lic = "" - if resources: - if resources.get("source_url"): - src = resources["source_url"] - if resources.get("license_url"): - lic = yaml_obj["resources"]["license_url"] - - manifest_obj["links"] = {"source_url": src, "license_url": lic} - - # Processing Instructions -- optional - # No sense of this in yaml... - manifest_obj["processing_instructions"] = [] - - return manifest_obj - - -def verify_yaml(file): - return file and re.match(r"^\S*.spec.yaml$", file) - - -def open_and_read_spec(file): - spec_file = open(file) - loaded_spec = yaml.safe_load(spec_file) - return loaded_spec - - -def write_to_json(obj, path): - with open(path + "/manifest.json", "w") as json_out: - json.dump(obj, json_out, indent=4) - - -if __name__ == "__main__": - main() diff --git a/tools/stats.py b/tools/stats.py deleted file mode 100755 index f9a4d7a94a..0000000000 --- a/tools/stats.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -import logging -import yaml - - -PLUGINS = 0 -TRIGGERS = 0 -ACTIONS = 0 -# Example or incomplete plugins -SKIP_PLUGINS = ["dbdemo", "example", "example_types"] -DEFAULT_TYPES = [ - "bool", - "boolean", - "bytes", - "credential_username_password", - "credential_secret_key", - "credential_asymmetric_key", - "date", - "file", - "float", - "int", - "integer", - "number", - "object", - "password", - "python", - "string", - "[]boolean", - "[]bytes", - "[]date", - "[]file", - "[]string", - "[]integer", - "[]object", -] - - -def read_plugin_spec(path): - if not os.path.isfile(path): - return None - f = open(path, "r") - return yaml.safe_load(f) - - -def check_args(args): - if len(args) != 2: - print("./stats.py [|total|go|python|custom_inputs]") - sys.exit(0) - return args[1] - - -def is_yaml(data): - if type(data) is None: - logging.error("Error: %s is not a file!", PATH) - sys.exit(1) - - -def get_specs(lang="total"): - spec_list = [] - for f in os.listdir("."): - if f in SKIP_PLUGINS: - continue - if os.path.isdir(f): - spec = f + "/plugin.spec.yaml" - if os.path.isfile(spec): - if lang == "go": - if ( - os.path.isdir(f + "/cmd") - or os.path.isdir(f + "/connection") - or os.path.isdir(f + "/actions") - or os.path.isdir(f + "/triggers") - ): - spec_list.append(spec) - if lang == "python": - if os.path.isdir(f + "/bin") or os.path.isdir(f + "/setup.py"): - spec_list.append(spec) - if lang == "total": - spec_list.append(spec) - return spec_list - - -def count_components(plugin, component=None): - if component not in plugin: - return 0 - if plugin[component] is None: - return 0 - return len(plugin[component]) - - -def list_components(plugin, component=None, behavior=None): - components = [] - if component not in plugin: - return components - if plugin[component] is None: - return components - for name in plugin[component]: - if behavior == "custom_inputs": - if "input" not in plugin[component][name]: - continue - if plugin[component][name]["input"] is None: - continue - for var in plugin[component][name]["input"]: - tipe = plugin[component][name]["input"][var]["type"] - if tipe in DEFAULT_TYPES: - continue - print(" Found non-builtin input type: {}".format(tipe)) - components.append(name) - else: - components.append(name) - return components - - -def gen_stats(plugin, behavior=None): - - triggers = list_components(plugin, component="triggers", behavior=behavior) - actions = list_components(plugin, component="actions", behavior=behavior) - - if behavior == "custom_inputs": - result = len(triggers) + len(actions) - if result == 0: - return - - print("Name: %s" % plugin.get("name")) - print(" Title: %s" % plugin.get("title")) - print(" Description: %s" % plugin.get("description")) - print(" Vendor: %s" % plugin.get("vendor")) - print(" Version: %s" % plugin.get("version")) - trigger_count = count_components(plugin, component="triggers") - action_count = count_components(plugin, component="actions") - print(" Triggers: %d" % trigger_count) - print(" Actions: %d" % action_count) - - if trigger_count > 0: - print(" Trigger List: %s" % " ".join(map(str, triggers))) - if action_count > 0: - print(" Action List: %s" % " ".join(map(str, actions))) - - -def gen_total_stats(lang): - global PLUGINS - global TRIGGERS - global ACTIONS - spec_list = get_specs(lang) - for path in spec_list: - data = read_plugin_spec(path) - is_yaml(data) - PLUGINS += 1 - TRIGGERS += count_components(data, component="triggers") - ACTIONS += count_components(data, component="actions") - gen_stats(data) - - print( - """ -Total: - Plugins: %d - Triggers: %d - Actions: %d - """ - % (PLUGINS, TRIGGERS, ACTIONS) - ) - - -def gen_custom_input_stats(): - spec_list = get_specs() - for path in spec_list: - data = read_plugin_spec(path) - is_yaml(data) - gen_stats(data, behavior="custom_inputs") - - -ARG = check_args(sys.argv) - -if ARG == "total": - gen_total_stats(ARG) - sys.exit(0) - -if ARG == "go": - gen_total_stats(ARG) - sys.exit(0) - -if ARG == "python": - gen_total_stats(ARG) - sys.exit(0) - -if ARG == "custom_inputs": - gen_custom_input_stats() - sys.exit(0) - -# Stats for single plugin -PATH = ARG -DATA = read_plugin_spec(PATH) -is_yaml(DATA) -gen_stats(DATA) -sys.exit(0) diff --git a/tools/templates/template.json_to_spec.j2 b/tools/templates/template.json_to_spec.j2 deleted file mode 100644 index 90d16c7309..0000000000 --- a/tools/templates/template.json_to_spec.j2 +++ /dev/null @@ -1,20 +0,0 @@ -types: - {% for item in sections["types"].items() -%} - {{ item[0] }}: - {%- for param in item[1].items() %} - {{ param[0] }}: - title: "{{ param[1].title }}" - type: {{ param[1].type }} - description: "{{ param[1].description }}" - required: false - {%- endfor %} - {% endfor %} - -output: - {% for param in sections["output"].items() -%} - {{ param[0] }}: - title: "{{ param[1].title }}" - type: {{ param[1].type }} - description: "{{ param[1].description }}" - required: false - {% endfor %} diff --git a/tools/update-tools.sh b/tools/update-tools.sh deleted file mode 100755 index 8b72579d2b..0000000000 --- a/tools/update-tools.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -platform=$(uname) - -if [[ "${platform}" == "Darwin" ]]; then - echo "[*] Executing update/installation for MacOS!" - echo "[*] Installing/updating jq via homebrew" - brew reinstall jq > /dev/null; true - -elif [[ "${platform}" == "Linux" ]] && [[ -f /etc/debian_version ]]; then - echo "[*] Executing update/installation for Debian Linux!" - echo "[*] Installing/updating jq..." - sudo apt-get -qq install -y jq - -elif [[ "${platform}" == "Linux" ]] && [[ -f /etc/redhat-release ]]; then - echo "[*] Executing update/installation for Red Hat Linux!" - echo "[*] Installing/updating jq..." - sudo yum install -q -y jq - -else - echo "[!] Unsupported OS found! Unable to install jq!" -fi - -echo "[*] Installing/updating insight-plugin..." -sudo -H python3 -m pip install --user --upgrade insight-plugin > /dev/null; true - -echo "[*] Installing InsightConnect validator tooling..." -sudo -H python3 -m pip install --user --upgrade insightconnect-integrations-validators > /dev/null; true - -echo "[*] Installing PyYAML..." -sudo -H python3 -m pip install --user --upgrade pyyaml > /dev/null; true - -echo "[*] Installing pre-commit..." -sudo -H python3 -m pip install --user --upgrade pre-commit > /dev/null; true - -echo "[*] Installing mdl..." -sudo gem install mdl > /dev/null; true - -echo "[*] Installing js-yaml..." -sudo npm install -g js-yaml > /dev/null; true - -echo "[*] Complete! Tooling installed & updated!"