diff --git a/.cookiecutter.json b/.cookiecutter.json index c5f93035..f68fe80f 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -21,7 +21,7 @@ "_drift_manager": { "template": "https://github.com/nautobot/cookiecutter-nautobot-app.git", "template_dir": "nautobot-app", - "template_ref": "refs/tags/nautobot-app-v2.3.0", + "template_ref": "refs/tags/nautobot-app-v2.3.1", "cookie_dir": "", "branch_prefix": "drift-manager", "pull_request_strategy": "create", @@ -29,7 +29,7 @@ "black" ], "draft": true, - "baked_commit_ref": "76744d296d468b03fb528c902f68c3dc230a9a3f" + "baked_commit_ref": "e24f60e18572850b86d48f95e084c441cfed6c56" } } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24473bd9..5f4b0900 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: # yamllint disable-line rule:truthy rule:comments pull_request: ~ env: - APP_NAME: "nautobot-app-chatops" + APP_NAME: "nautobot-chatops" jobs: ruff-format: @@ -91,6 +91,10 @@ jobs: uses: "actions/checkout@v4" - name: "Setup environment" uses: "networktocode/gh-action-setup-poetry-environment@v6" + - name: "Constrain Nautobot version and regenerate lock file" + env: + INVOKE_NAUTOBOT_DEV_EXAMPLE_LOCAL: "true" + run: "poetry run invoke lock --constrain-nautobot-ver --constrain-python-ver" - name: "Set up Docker Buildx" id: "buildx" uses: "docker/setup-buildx-action@v3" @@ -108,6 +112,7 @@ jobs: build-args: | NAUTOBOT_VER=${{ matrix.nautobot-version }} PYTHON_VER=${{ matrix.python-version }} + CI=true - name: "Copy credentials" run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" @@ -122,14 +127,14 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] db-backend: ["postgresql"] nautobot-version: ["stable"] include: - python-version: "3.11" db-backend: "postgresql" nautobot-version: "2.0.0" - - python-version: "3.11" + - python-version: "3.12" db-backend: "mysql" nautobot-version: "stable" runs-on: "ubuntu-22.04" @@ -158,6 +163,7 @@ jobs: build-args: | NAUTOBOT_VER=${{ matrix.nautobot-version }} PYTHON_VER=${{ matrix.python-version }} + CI=true - name: "Copy credentials" run: "cp development/creds.example.env development/creds.env" - name: "Use Mysql invoke settings when needed" @@ -195,7 +201,7 @@ jobs: - name: "Set up Python" uses: "actions/setup-python@v5" with: - python-version: "3.11" + python-version: "3.12" - name: "Install Python Packages" run: "pip install poetry" - name: "Set env" @@ -230,7 +236,7 @@ jobs: - name: "Set up Python" uses: "actions/setup-python@v5" with: - python-version: "3.11" + python-version: "3.12" - name: "Install Python Packages" run: "pip install poetry" - name: "Set env" diff --git a/development/Dockerfile b/development/Dockerfile index 69d8a0f9..cf1ca1b3 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -53,29 +53,18 @@ RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \ WORKDIR /source COPY . /source -# Get container's installed Nautobot version as a forced constraint -# NAUTOBOT_VER may be a branch name and not a published release therefor we need to get the installed version -# so pip can use it to recognize local constraints. -RUN pip show nautobot | grep "^Version: " | sed -e 's/Version: /nautobot==/' > constraints.txt +# Build args must be declared in each stage +ARG PYTHON_VER -# Use Poetry to grab dev dependencies from the lock file -# Can be improved in Poetry 1.2 which allows `poetry install --only dev` -# -# We can't use the entire freeze as it takes forever to resolve with rigidly fixed non-direct dependencies, -# especially those that are only direct to Nautobot but the container included versions slightly mismatch -RUN poetry export -f requirements.txt --without-hashes --extras all --output poetry_freeze_base.txt -RUN poetry export -f requirements.txt --without-hashes --extras all --with dev --output poetry_freeze_all.txt -RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_dev.txt - -# Install all local project as editable, constrained on Nautobot version, to get any additional -# direct dependencies of the app -RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ - pip install -c constraints.txt -e .[all] +# Constrain the Nautobot version to NAUTOBOT_VER +# In CI, this should be done outside of the Dockerfile to prevent cross-compile build failures +ARG CI +RUN if [ -z "${CI+x}" ]; then \ + INSTALLED_NAUTOBOT_VER=$(pip show nautobot | grep "^Version" | sed "s/Version: //"); \ + poetry add --lock nautobot@${INSTALLED_NAUTOBOT_VER} --python ${PYTHON_VER}; fi -# Install any dev dependencies frozen from Poetry -# Can be improved in Poetry 1.2 which allows `poetry install --only dev` -RUN --mount=type=cache,target="/root/.cache/pip",sharing=locked \ - pip install -c constraints.txt -r poetry_freeze_dev.txt +# Install the app +RUN poetry install --extras all --with dev COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py # !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/app_config_schema.py b/development/app_config_schema.py index a779b14e..e52e2478 100644 --- a/development/app_config_schema.py +++ b/development/app_config_schema.py @@ -40,7 +40,9 @@ def _main(): **SchemaBuilder().to_json_schema(app_config), # type: ignore } app_config = import_module(package_name).config - _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings) + _enrich_object_schema( + schema, app_config.default_settings, app_config.required_settings + ) schema_path.write_text(json.dumps(schema, indent=4) + "\n") print(f"\n==================\nGenerated schema:\n\n{schema_path}\n") print( diff --git a/development/nautobot_config.py b/development/nautobot_config.py index 31cc69c5..12a886c0 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -18,8 +18,12 @@ if "debug_toolbar" not in INSTALLED_APPS: # noqa: F405 INSTALLED_APPS.append("debug_toolbar") # noqa: F405 - if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE: # noqa: F405 - MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa: F405 + if ( + "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE + ): # noqa: F405 + MIDDLEWARE.insert( + 0, "debug_toolbar.middleware.DebugToolbarMiddleware" + ) # noqa: F405 # # Misc. settings @@ -51,7 +55,9 @@ "NAUTOBOT_DB_PORT", default_db_settings[nautobot_db_engine]["NAUTOBOT_DB_PORT"], ), # Database port, default to postgres - "CONN_MAX_AGE": int(os.getenv("NAUTOBOT_DB_TIMEOUT", "300")), # Database timeout + "CONN_MAX_AGE": int( + os.getenv("NAUTOBOT_DB_TIMEOUT", "300") + ), # Database timeout "ENGINE": nautobot_db_engine, } } diff --git a/docs/requirements.txt b/docs/requirements.txt index ca55a90d..bf10c13b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ -mkdocs==1.5.2 -mkdocs-material==9.1.15 +mkdocs==1.6.0 +mkdocs-material==9.5.32 markdown-version-annotations==1.0.1 -mkdocstrings-python==1.5.2 -mkdocstrings==0.22.0 -mkdocs-include-markdown-plugin==6.0.3 +griffe==1.1.1 +mkdocstrings-python==1.10.8 +mkdocstrings==0.25.2 diff --git a/invoke.example.yml b/invoke.example.yml index 3d718b24..8d6a3af1 100644 --- a/invoke.example.yml +++ b/invoke.example.yml @@ -1,18 +1,15 @@ --- nautobot_chatops: - project_name: "nautobot-chatops" nautobot_ver: "2.0.0" - local: false python_ver: "3.11" - compose_dir: "development" - compose_files: - - "docker-compose.base.yml" - - "docker-compose.redis.yml" - - "docker-compose.postgres.yml" - - "mattermost/docker-compose.yml" - - "ansible/docker-compose.yml" - - "docker-compose.dev.yml" - # Uncomment below if using Slack Socket Mode - # - "docker-compose.socket.yml" - # Uncomment below if using Microsoft Bot Framework Emulator - # - "docker-compose.bot-framework.yml" + # local: false + # compose_dir: "/full/path/to/nautobot-app-chatops/development" + +# The following is an example of using MySQL as the database backend +# --- +# nautobot_chatops: +# compose_files: +# - "docker-compose.base.yml" +# - "docker-compose.redis.yml" +# - "docker-compose.mysql.yml" +# - "docker-compose.dev.yml" diff --git a/mkdocs.yml b/mkdocs.yml index c806a12c..0028d3d2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,6 @@ --- dev_addr: "127.0.0.1:8001" -edit_uri: "edit/main/nautobot-app-chatops/docs" +edit_uri: "edit/main/docs" site_dir: "nautobot_chatops/static/nautobot_chatops/docs" site_name: "Nautobot ChatOps App Documentation" site_url: "https://docs.nautobot.com/projects/chatops/en/latest/" diff --git a/nautobot_chatops/tests/test_basic.py b/nautobot_chatops/tests/test_basic.py index d72f2d02..83142fac 100644 --- a/nautobot_chatops/tests/test_basic.py +++ b/nautobot_chatops/tests/test_basic.py @@ -11,11 +11,21 @@ class TestDocsPackaging(unittest.TestCase): def test_version(self): """Verify that pyproject.toml dev dependencies have the same versions as in the docs requirements.txt.""" - parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + parent_path = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + ) poetry_path = os.path.join(parent_path, "pyproject.toml") - poetry_details = toml.load(poetry_path)["tool"]["poetry"]["group"]["dev"]["dependencies"] - with open(f"{parent_path}/docs/requirements.txt", "r", encoding="utf-8") as file: - requirements = [line for line in file.read().splitlines() if (len(line) > 0 and not line.startswith("#"))] + poetry_details = toml.load(poetry_path)["tool"]["poetry"]["group"]["dev"][ + "dependencies" + ] + with open( + f"{parent_path}/docs/requirements.txt", "r", encoding="utf-8" + ) as file: + requirements = [ + line + for line in file.read().splitlines() + if (len(line) > 0 and not line.startswith("#")) + ] for pkg in requirements: package_name = pkg if len(pkg.split("==")) == 2: # noqa: PLR2004 diff --git a/pyproject.toml b/pyproject.toml index b57ff6f2..6f3ed22f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] packages = [ { include = "nautobot_chatops" }, @@ -39,22 +40,8 @@ include = [ "nso" = "nautobot_chatops.integrations.nso.worker:nso" [tool.poetry.dependencies] -Markdown = "!=3.3.5" -PyJWT = "^2.1.0" -PyYAML = { version = "^6.0", optional = true } -aiodns = "^1.0" -aiohttp = "^3.7.3" -asgiref = "^3.4.1" -certifi = { version = ">=2021.5.30", optional = true } -cloudvision = { version = "^1.1", optional = true } -cvprac = { version = "^1.0.6", optional = true } -defusedxml = { version = "^0.7.1", optional = true } -diffsync = { version = "^1.3.0", optional = true } -ipaddr = { version = "^2.2.0", optional = true } -ipfabric = { version = "~6.0.9", optional = true } -ipfabric-diagrams = { version = "~6.0.2", optional = true } -isodate = { version = "^0.6.1", optional = true } -meraki = { version = ">=1.7.2,<=1.45.0", optional = true } +python = ">=3.8,<3.13" +# Used for local development nautobot = "^2.0.0" nautobot-capacity-metrics = "^3.0.0" netmiko = { version = "^4.0.0", optional = true } @@ -80,21 +67,13 @@ Markdown = "*" # Render custom markdown for version added/changed/remove notes markdown-version-annotations = "1.0.1" # Rendering docs to HTML -mkdocs = "1.5.2" -mkdocs-include-markdown-plugin = "6.0.3" +mkdocs = "1.6.0" # Material for MkDocs theme -mkdocs-material = "9.1.15" +mkdocs-material = "9.5.32" # Automatic documentation from sources, for MkDocs -mkdocstrings = "0.22.0" -mkdocstrings-python = "1.5.2" -prybar = "*" -pylint = "*" -pylint-django = "*" -pylint-nautobot = "*" -requests-mock = "^1.9.3" -ruff = "0.5.5" -yamllint = "*" -toml = "*" +mkdocstrings = "0.25.2" +mkdocstrings-python = "1.10.8" +griffe = "1.1.1" towncrier = "~23.6.0" to-json-schema = "*" jsonschema = "*" @@ -160,12 +139,12 @@ nautobot = ["nautobot"] [tool.pylint.master] # Include the pylint_django plugin to avoid spurious warnings about Django patterns -load-plugins="pylint_django, pylint_nautobot" -ignore=".venv" +load-plugins = "pylint_django, pylint_nautobot" +ignore = ".venv" [tool.pylint.basic] # No docstrings required for private methods (Pylint default), or for test_ functions, or for inner Meta classes. -no-docstring-rgx="^(_|test_|Meta$)" +no-docstring-rgx = "^(_|test_|Meta$)" [tool.pylint.messages_control] disable = [ @@ -201,26 +180,27 @@ target-version = "py38" select = [ "D", # pydocstyle "F", "E", "W", # flake8 + "PL", # pylint "S", # bandit "I", # isort ] ignore = [ # warning: `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible. - "D203", # 1 blank line required before class docstring + "D203", # 1 blank line required before class docstring # D212 is enabled by default in google convention, and complains if we have a docstring like: # """ # My docstring is on the line after the opening quotes instead of on the same line as them. # """ # We've discussed and concluded that we consider this to be a valid style choice. - "D212", # Multi-line docstring summary should start at the first line - "D213", # Multi-line docstring summary should start at the second line + "D212", # Multi-line docstring summary should start at the first line + "D213", # Multi-line docstring summary should start at the second line # Produces a lot of issues in the current codebase. - "D401", # First line of docstring should be in imperative mood - "D407", # Missing dashed underline after section - "D416", # Section name ends in colon - "E501", # Line too long + "D401", # First line of docstring should be in imperative mood + "D407", # Missing dashed underline after section + "D416", # Section name ends in colon + "E501", # Line too long ] [tool.ruff.lint.pydocstyle] diff --git a/tasks.py b/tasks.py index 1d34c5e2..4efdb6ef 100644 --- a/tasks.py +++ b/tasks.py @@ -13,10 +13,12 @@ """ import os +import re from pathlib import Path from time import sleep from invoke.collection import Collection +from invoke.exceptions import Exit from invoke.tasks import task as invoke_task @@ -48,7 +50,7 @@ def is_truthy(arg): namespace.configure( { "nautobot_chatops": { - "nautobot_ver": "2.0.0", + "nautobot_ver": "2.3.1", "project_name": "nautobot-chatops", "python_ver": "3.11", "local": False, @@ -72,7 +74,9 @@ def _is_compose_included(context, name): def _await_healthy_service(context, service): - container_id = docker_compose(context, f"ps -q -- {service}", pty=False, echo=False, hide=True).stdout.strip() + container_id = docker_compose( + context, f"ps -q -- {service}", pty=False, echo=False, hide=True + ).stdout.strip() _await_healthy_container(context, container_id) @@ -132,7 +136,9 @@ def docker_compose(context, command, **kwargs): ] for compose_file in context.nautobot_chatops.compose_files: - compose_file_path = os.path.join(context.nautobot_chatops.compose_dir, compose_file) + compose_file_path = os.path.join( + context.nautobot_chatops.compose_dir, compose_file + ) compose_command_tokens.append(f' -f "{compose_file_path}"') compose_command_tokens.append(command) @@ -171,7 +177,9 @@ def run_command(context, command, **kwargs): if "nautobot" in results.stdout: compose_command = f"exec{command_env_args} nautobot {command}" else: - compose_command = f"run{command_env_args} --rm --entrypoint='{command}' nautobot" + compose_command = ( + f"run{command_env_args} --rm --entrypoint='{command}' nautobot" + ) pty = kwargs.pop("pty", True) @@ -207,17 +215,63 @@ def generate_packages(context): run_command(context, command) +def _get_docker_nautobot_version(context, nautobot_ver=None, python_ver=None): + """Extract Nautobot version from base docker image.""" + if nautobot_ver is None: + nautobot_ver = context.nautobot_chatops.nautobot_ver + if python_ver is None: + python_ver = context.nautobot_chatops.python_ver + dockerfile_path = os.path.join(context.nautobot_chatops.compose_dir, "Dockerfile") + base_image = ( + context.run(f"grep --max-count=1 '^FROM ' {dockerfile_path}", hide=True) + .stdout.strip() + .split(" ")[1] + ) + base_image = base_image.replace(r"${NAUTOBOT_VER}", nautobot_ver).replace( + r"${PYTHON_VER}", python_ver + ) + pip_nautobot_ver = context.run( + f"docker run --rm --entrypoint '' {base_image} pip show nautobot", hide=True + ) + match_version = re.search( + r"^Version: (.+)$", pip_nautobot_ver.stdout.strip(), flags=re.MULTILINE + ) + if match_version: + return match_version.group(1) + else: + raise Exit(f"Nautobot version not found in Docker base image {base_image}.") + + @task( help={ "check": ( "If enabled, check for outdated dependencies in the poetry.lock file, " "instead of generating a new one. (default: disabled)" - ) + ), + "constrain_nautobot_ver": ( + "Run 'poetry add nautobot@[version] --lock' to generate the lockfile, " + "where [version] is the version installed in the Dockerfile's base image. " + "Generally intended to be used in CI and not for local development. (default: disabled)" + ), + "constrain_python_ver": ( + "When using `constrain_nautobot_ver`, further constrain the nautobot version " + "to python_ver so that poetry doesn't complain about python version incompatibilities. " + "Generally intended to be used in CI and not for local development. (default: disabled)" + ), } ) -def lock(context, check=False): - """Generate poetry.lock inside the Nautobot container.""" - run_command(context, f"poetry {'check' if check else 'lock --no-update'}") +def lock( + context, check=False, constrain_nautobot_ver=False, constrain_python_ver=False +): + """Generate poetry.lock file.""" + if constrain_nautobot_ver: + docker_nautobot_version = _get_docker_nautobot_version(context) + command = f"poetry add --lock nautobot@{docker_nautobot_version}" + if constrain_python_ver: + command += f" --python {context.nautobot_chatops.python_ver}" + else: + command = f"poetry {'check' if check else 'lock --no-update'}" + run_command(context, command) # ------------------------------------------------------------------------------ @@ -248,7 +302,9 @@ def restart(context, service=""): def stop(context, service=""): """Stop specified or all services, if service is not specified, remove all containers.""" print("Stopping Nautobot...") - docker_compose(context, "stop" if service else "down --remove-orphans", service=service) + docker_compose( + context, "stop" if service else "down --remove-orphans", service=service + ) @task( @@ -267,7 +323,9 @@ def destroy(context, volumes=True, import_db_file=""): return if not volumes: - raise ValueError("Cannot specify `--no-volumes` and `--import-db-file` arguments at the same time.") + raise ValueError( + "Cannot specify `--no-volumes` and `--import-db-file` arguments at the same time." + ) print(f"Importing database file: {import_db_file}...") @@ -284,12 +342,16 @@ def destroy(context, volumes=True, import_db_file=""): "db", ] - container_id = docker_compose(context, " ".join(command), pty=False, echo=False, hide=True).stdout.strip() + container_id = docker_compose( + context, " ".join(command), pty=False, echo=False, hide=True + ).stdout.strip() _await_healthy_container(context, container_id) print("Stopping database container...") context.run(f"docker stop {container_id}", pty=False, echo=False, hide=True) - print("Database import complete, you can start Nautobot with the following command:") + print( + "Database import complete, you can start Nautobot with the following command:" + ) print("invoke start") @@ -461,7 +523,9 @@ def dbshell(context, db_name="", input_file="", output_file="", query=""): if input_file and query: raise ValueError("Cannot specify both, `input_file` and `query` arguments") if output_file and not (input_file or query): - raise ValueError("`output_file` argument requires `input_file` or `query` argument") + raise ValueError( + "`output_file` argument requires `input_file` or `query` argument" + ) env = {} if query: @@ -599,7 +663,9 @@ def backup_db(context, db_name="", output_file="dump.sql", readable=True): docker_compose(context, " ".join(command), pty=False) print(50 * "=") - print("The database backup has been successfully completed and saved to the following file:") + print( + "The database backup has been successfully completed and saved to the following file:" + ) print(output_file) print("You can import this database backup with the following command:") print(f"invoke import-db --input-file '{output_file}'") @@ -838,51 +904,3 @@ def validate_app_config(context): file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "validate"}, ) - - -# ------------------------------------------------------------------------------ -# APP CUSTOM -# ------------------------------------------------------------------------------ -@task -def bootstrap_mattermost(context): - """Bootstrap Nautobot data to be used with Mattermost.""" - nbshell(context, file="development/mattermost/nautobot_bootstrap.py") - - -@task -def backup_mattermost(context): - """Export Mattermost data to the SQL file. Certain tables are ignored.""" - output = "./development/mattermost/dump.sql" - - command = [ - "exec", - "--", - "mattermost", - "bash", - "-c", - "'", - "pg_dump", - "--inserts", - "--user=$POSTGRES_USER", - "--dbname=$POSTGRES_DB", - "'", - f"> {output}", - ] - - docker_compose(context, " ".join(command)) - - -@task -def connect_awx_container(context, container_name="tools_awx_1"): - """Connect nautobot and celery containers to awx container. - - Bridge network is defined in `development/ansible/docker-compose.yaml`. - To run testing awx instance, follow [instructions] - (https://github.com/ansible/awx/tree/devel/tools/docker-compose#getting-started) - Before running `make docker-compose` comment out `- 8080:8080` port mapping in file - `tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2` to avoid port conflict with nautobot. - After setting up awx, cd back to chatops repo and run `invoke connect-awx-container`. - """ - bridge_network = f"{context.nautobot_chatops.project_name}_awx" - context.run(f"docker network connect --alias awx {bridge_network} {container_name}") - print(f"Container {container_name} connected to {bridge_network} network")