Refactor: Move stuff around, clean logging

ansible_collections/arista/avd/plugins/__init__.py

@@ -19,6 +19,6 @@
 
     display = Display()
 
-    display.warning(f"AVD detected it is running from source, prepending the path to the source of pyavd '{str(PYTHON_AVD_PATH)}' to PATH to use it.")
+    display.warning(f"AVD detected it is running from source, prepending the path to the source of pyavd '{PYTHON_AVD_PATH}' to PATH to use it.")
 
     sys.path = [str(PYTHON_AVD_PATH), *sys.path]

ansible_collections/arista/avd/plugins/plugin_utils/utils/init_logging.py

@@ -5,14 +5,30 @@
 
 import logging
 
+from ansible.utils.display import Display
+
 from .python_to_ansible_logging_handler import PythonToAnsibleHandler
 
 
 def init_pyavd_logging() -> None:
     """Specify logger parameters for pyavd."""
     pyavd_logger = logging.getLogger("pyavd")
+    schema_tools_logger = logging.getLogger("schema_tools")
+
     pyavd_handler = PythonToAnsibleHandler(None)
     pyavd_formatter = logging.Formatter("[pyavd] - %(message)s")
     pyavd_handler.setFormatter(pyavd_formatter)
+
     pyavd_logger.addHandler(pyavd_handler)
-    pyavd_logger.setLevel(logging.INFO)
+    schema_tools_logger.addHandler(pyavd_handler)
+
+    verbosity = Display().verbosity
+    if verbosity > 3:
+        pyavd_logger.setLevel(logging.DEBUG)
+        schema_tools_logger.setLevel(logging.DEBUG)
+    elif verbosity > 0:
+        pyavd_logger.setLevel(logging.INFO)
+        schema_tools_logger.setLevel(logging.INFO)
+    else:
+        pyavd_logger.setLevel(logging.WARNING)
+        schema_tools_logger.setLevel(logging.WARNING)

python-avd/pyavd/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2023-2024 Arista Networks, Inc.
 # Use of this source code is governed by the Apache License 2.0
 # that can be found in the LICENSE file.
-from ._utils.check_running_from_source import check_running_from_source
+from ._utils.compile_templates import check_templates
 from .get_avd_facts import get_avd_facts
 from .get_device_config import get_device_config
 from .get_device_doc import get_device_doc
@@ -30,4 +30,4 @@
     "ValidationResult",
 ]
 
-check_running_from_source()
+check_templates()

python-avd/pyavd/_schema/__init__.py

@@ -1,3 +1,6 @@
 # Copyright (c) 2023-2024 Arista Networks, Inc.
 # Use of this source code is governed by the Apache License 2.0
 # that can be found in the LICENSE file.
+from .build_schemas import check_schemas
+
+check_schemas()

python-avd/pyavd/_schema/build_schemas.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2023-2024 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the LICENSE file.
+import logging
+
+from pyavd._utils.hash_dir import check_hash
+from pyavd.constants import RUNNING_FROM_SRC
+
+from .constants import (
+    EOS_CLI_CONFIG_GEN_SCHEMA_DIR,
+    # METASCHEMA_DIR,
+    EOS_DESIGNS_SCHEMA_DIR,
+)
+
+LOGGER = logging.getLogger(__name__)
+
+
+def check_schemas() -> None:
+    """TODO.
+
+    Schemas
+    """
+    if not RUNNING_FROM_SRC:
+        return
+
+    LOGGER.info("pyavd running from source detected, checking schemas for any changes...")
+
+    # TODO: ok to import from here
+    from schema_tools.build_schemas import build_schemas
+
+    # eos_designs
+    eos_designs_changed, eos_designs_new_hash = check_hash(EOS_DESIGNS_SCHEMA_DIR / "schema_fragments")
+    eos_cli_config_gen_changed, eos_cli_config_gen_new_hash = check_hash(EOS_CLI_CONFIG_GEN_SCHEMA_DIR / "schema_fragments")
+    if eos_designs_changed or eos_cli_config_gen_changed:
+        LOGGER.info("Recompiling schemas...")
+        build_schemas()
+        with (EOS_DESIGNS_SCHEMA_DIR / "schema_fragments/.hash").open("w") as fd:
+            fd.write(eos_designs_new_hash)
+        with (EOS_CLI_CONFIG_GEN_SCHEMA_DIR / "schema_fragments/.hash").open("w") as fd:
+            fd.write(eos_cli_config_gen_new_hash)

python-avd/pyavd/_utils/compile_templates.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2023-2024 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the LICENSE file.
+import logging
+
+from pyavd.constants import (
+    EOS_CLI_CONFIG_GEN_JINJA2_PRECOMPILED_TEMPLATE_PATH,
+    EOS_CLI_CONFIG_GEN_JINJA2_TEMPLATE_PATH,
+    EOS_DESIGNS_JINJA2_PRECOMPILED_TEMPLATE_PATH,
+    EOS_DESIGNS_JINJA2_TEMPLATE_PATH,
+    RUNNING_FROM_SRC,
+)
+from pyavd.templater import Templar
+
+from .hash_dir import check_hash
+
+LOGGER = logging.getLogger(__name__)
+
+
+def compile_eos_cli_config_gen_templates() -> None:
+    """TODO."""
+    templar = Templar(precompiled_templates_path=EOS_CLI_CONFIG_GEN_JINJA2_PRECOMPILED_TEMPLATE_PATH, searchpaths=[EOS_CLI_CONFIG_GEN_JINJA2_TEMPLATE_PATH])
+    templar.compile_templates_in_paths(
+        precompiled_templates_path=EOS_CLI_CONFIG_GEN_JINJA2_PRECOMPILED_TEMPLATE_PATH, searchpaths=[EOS_CLI_CONFIG_GEN_JINJA2_TEMPLATE_PATH]
+    )
+
+
+def compile_eos_designs_templates() -> None:
+    """TODO."""
+    templar = Templar(precompiled_templates_path=EOS_DESIGNS_JINJA2_PRECOMPILED_TEMPLATE_PATH, searchpaths=[EOS_DESIGNS_JINJA2_TEMPLATE_PATH])
+    templar.compile_templates_in_paths(precompiled_templates_path=EOS_DESIGNS_JINJA2_PRECOMPILED_TEMPLATE_PATH, searchpaths=[EOS_DESIGNS_JINJA2_TEMPLATE_PATH])
+
+
+def _check_eos_designs_templates() -> None:
+    """TODO.
+
+    Templates
+    """
+    if not RUNNING_FROM_SRC:
+        return
+
+    LOGGER.info("pyavd running from source detected, checking eos_designs templates for any changes...")
+    dir_path = EOS_DESIGNS_JINJA2_TEMPLATE_PATH
+
+    changed, new_hash = check_hash(dir_path)
+    if changed:
+        LOGGER.info("Recompiling eos_designs templates...")
+        compile_eos_designs_templates()
+        with (dir_path / ".hash").open("w") as fd:
+            fd.write(new_hash)
+
+
+def _check_eos_cli_config_gen_templates() -> None:
+    """TODO.
+
+    Templates
+    """
+    if not RUNNING_FROM_SRC:
+        return
+
+    LOGGER.info("pyavd running from source detected, checking eos_cli_config_gen templates for any changes...")
+    dir_path = EOS_CLI_CONFIG_GEN_JINJA2_TEMPLATE_PATH
+
+    changed, new_hash = check_hash(dir_path)
+    if changed:
+        LOGGER.info("Recompiling eos_cli_config_gen templates...")
+        compile_eos_cli_config_gen_templates()
+        with (dir_path / ".hash").open("w") as fd:
+            fd.write(new_hash)
+
+
+def check_templates() -> None:
+    """TODO."""
+    _check_eos_designs_templates()
+    _check_eos_cli_config_gen_templates()

python-avd/schema_tools/build_schemas.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2023-2024 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the LICENSE file.
+import logging
+from pathlib import Path
+from textwrap import indent
+
+import jsonschema
+from deepmerge import always_merger
+from yaml import CSafeDumper, CSafeLoader
+from yaml import dump as yaml_dump
+from yaml import load as yaml_load
+
+from .constants import DOCS_PATHS, LICENSE_HEADER, SCHEMA_FRAGMENTS_PATHS, SCHEMA_PATHS
+from .generate_docs.mdtabsgen import get_md_tabs
+from .metaschema.meta_schema_model import AristaAvdSchema
+from .store import create_store
+
+FRAGMENTS_PATTERN = "*.yml"
+
+LOGGER = logging.getLogger(__name__)
+
+
+def combine_schemas() -> None:
+    """Combine all schema fragments into a single YAML file."""
+    for schema_name, fragments_path in SCHEMA_FRAGMENTS_PATHS.items():
+        LOGGER.info("Combining fragments %s", fragments_path)
+        if schema_name not in SCHEMA_PATHS:
+            msg = f"Invalid schema name '{schema_name}'"
+            raise KeyError(msg)
+
+        schema = {}
+        for fragment_filename in sorted(fragments_path.glob(FRAGMENTS_PATTERN)):
+            with fragment_filename.open(mode="r", encoding="UTF-8") as fragment_stream:
+                schema = always_merger.merge(schema, yaml_load(fragment_stream, Loader=CSafeLoader))
+
+        with SCHEMA_PATHS[schema_name].open(mode="w", encoding="UTF-8") as schema_stream:
+            schema_stream.write(indent(LICENSE_HEADER, prefix="# ") + "\n")
+            schema_stream.write(
+                "# yaml-language-server: $schema=../../_schema/avd_meta_schema.json\n"
+                "# Line above is used by RedHat's YAML Schema vscode extension\n"
+                "# Use Ctrl + Space to get suggestions for every field. Autocomplete will pop up after typing 2 letters.\n",
+            )
+            schema_stream.write(yaml_dump(schema, Dumper=CSafeDumper, sort_keys=False))
+
+
+def validate_schemas(schema_store: dict) -> None:
+    """Validate schemas according to metaschema."""
+    schema_validator = jsonschema.Draft7Validator(schema_store["avd_meta_schema"])
+    for schema_name in SCHEMA_FRAGMENTS_PATHS:
+        LOGGER.info("Validating schema '%s'", schema_name)
+        schema_validator.validate(schema_store[schema_name])
+
+
+def build_schema_tables(schema_store: dict) -> None:
+    """Build schema tables."""
+    for schema_name in SCHEMA_PATHS:
+        if schema_name not in SCHEMA_FRAGMENTS_PATHS:
+            continue
+
+        schema = AristaAvdSchema(**schema_store[schema_name])
+        table_names = sorted(schema._descendant_tables)
+        output_dir = DOCS_PATHS[schema_name].joinpath("tables")
+        for table_name in table_names:
+            LOGGER.debug("Building table: %s from schema %s", table_name, schema_name)
+            table_file = output_dir.joinpath(f"{table_name}.md")
+            with Path(table_file).open(mode="w", encoding="UTF-8") as file:
+                file.write(get_md_tabs(schema, table_name))
+
+        # Clean up other markdown files not covered by the tables.
+        remove_files = [file for file in output_dir.glob("*.md") if file.is_file() and file.name.removesuffix(".md") not in table_names]
+        for file in remove_files:
+            LOGGER.info("Deleting file %s", file.absolute())
+            file.unlink()
+
+
+def build_schemas() -> None:
+    """Combines the schema fragments, and rebuild the pickled schemas."""
+    combine_schemas()
+    LOGGER.info("Rebuilding pickled schemas")
+    schema_store = create_store(force_rebuild=True)
+    validate_schemas(schema_store)
+    LOGGER.info("Rebuilding schemas documentation tables...")
+    build_schema_tables(schema_store)