From a60e78a4f32853c36072656ca254cfad9aea5c8c Mon Sep 17 00:00:00 2001 From: KYTG Date: Thu, 8 Feb 2024 14:15:44 -0500 Subject: [PATCH] Convert tox to nox --- .github/workflows/flake8.yaml | 10 +- .github/workflows/test.yaml | 66 +++++++++---- .pre-commit-config.yaml | 64 +++++++++++++ noxfile.py | 175 ++++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++ tox.ini | 72 -------------- 6 files changed, 300 insertions(+), 95 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 noxfile.py create mode 100644 pyproject.toml delete mode 100644 tox.ini diff --git a/.github/workflows/flake8.yaml b/.github/workflows/flake8.yaml index b7b33ad..7c48c87 100644 --- a/.github/workflows/flake8.yaml +++ b/.github/workflows/flake8.yaml @@ -11,11 +11,11 @@ jobs: - name: Setup python for flake8 uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - uses: actions/checkout@v3 - - name: Install tox - run: python -m pip install tox + - name: Install nox + run: python -m pip install nox - name: Setup flake8 - run: tox --notest -e flake8 + run: nox --force-color -e flake8 --install-only - name: Run flake8 - run: tox -e flake8 + run: nox --force-color -e flake8 -vv diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0398d22..567361b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,9 +1,28 @@ name: test on: + workflow_dispatch: push: pull_request: jobs: + pre-commit: + name: pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pre-commit + pre-commit install + - name: Run pre-commit + run: pre-commit run --all-files + test: name: test ${{ matrix.py }} - ${{ matrix.netapi }} - ${{ matrix.salt }} runs-on: ubuntu-20.04 @@ -11,38 +30,49 @@ jobs: fail-fast: false matrix: py: - - "3.7" - "3.8" - "3.9" - "3.10" + - "3.11" netapi: - "cherrypy" - "tornado" salt: - - "v3004.2" - - "v3005.1" - - "v3006.0" - - "master" - exclude: - - py: "3.10" - salt: "v3004.2" - - py: "3.10" - salt: "v3005.1" + - "v3006.5" + steps: - - name: Setup python for test ${{ matrix.py }} + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.py }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} - - uses: actions/checkout@v3 + - name: Install setuptools_scm run: python -m pip install setuptools_scm - - name: Install tox - run: python -m pip install tox + - name: Install dependencies run: sudo apt update && sudo apt install -y libc6-dev libffi-dev gcc git openssh-server libzmq3-dev env: DEBIAN_FRONTEND: noninteractive - - name: Setup tests - run: tox --notest -e py${{ matrix.py }}-${{ matrix.netapi }}-${{ matrix.salt }} - - name: Run tests - run: tox -e py${{ matrix.py }}-${{ matrix.netapi }}-${{ matrix.salt }} + + - name: Install Nox + run: | + python -m pip install --upgrade pip + pip install nox + + - name: Install Test Requirements + env: + SALT_REQUIREMENT: salt==${{ matrix.salt }} + PYTHON_VERSIONS: ${{ matrix.py }} + API_BACKEND: ${{ matrix.netapi }} + run: | + nox --force-color -e tests --install-only + + - name: Test + env: + SALT_REQUIREMENT: salt==${{ matrix.salt }} + PYTHON_VERSIONS: ${{ matrix.py }} + API_BACKEND: ${{ matrix.netapi }} + SKIP_REQUIREMENTS_INSTALL: YES + run: | + nox --force-color -e tests -- -vv tests/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..90fe227 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,64 @@ +--- +minimum_pre_commit_version: 3.5.0 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-merge-conflict # Check for files that contain merge conflict strings. + - id: trailing-whitespace # Trims trailing whitespace. + args: [--markdown-linebreak-ext=md] + - id: mixed-line-ending # Replaces or checks mixed line ending. + args: [--fix=lf] + - id: end-of-file-fixer # Makes sure files end in a newline and only a newline. + - id: check-merge-conflict # Check for files that contain merge conflict strings. + - id: check-ast # Simply check whether files parse as valid python. + + # ----- Formatting ----------------------------------------------------------------------------> + - repo: https://github.com/myint/autoflake + rev: v2.2.1 + hooks: + - id: autoflake + name: Remove unused variables and imports + language: python + args: ["--in-place", "--remove-all-unused-imports", "--remove-unused-variables", "--expand-star-imports"] + files: \.py$ + + - repo: https://github.com/saltstack/pre-commit-remove-import-headers + rev: 1.1.0 + hooks: + - id: remove-import-headers + + - repo: https://github.com/s0undt3ch/salt-rewrite + # Automatically rewrite code with known rules + rev: 2.4.2 + hooks: + - id: salt-rewrite + alias: rewrite-tests + name: Rewrite the test suite + files: ^tests/.*\.py$ + args: [--silent, -E, fix_docstrings] + + - repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + name: Rewrite Code to be Py3+ + args: [ + --py3-plus + ] + + - repo: https://github.com/asottile/reorder_python_imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + args: [ + --py3-plus, + ] + + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + args: [-l 100] + + # <---- Formatting ----------------------------------------------------------------------------- diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..432929d --- /dev/null +++ b/noxfile.py @@ -0,0 +1,175 @@ +import datetime +import os +import pathlib +import shutil + +import nox +from nox.command import CommandFailed + + +API_BACKEND = [os.environ.get("API_BACKEND")] or ["cherrypy", "tornado"] +PYTHON_VERSIONS = [os.environ.get("PYTHON_VERSIONS")] or ["3.8", "3.9", "3.10", "3.11"] + +# Nox options +# Reuse existing virtualenvs +nox.options.reuse_existing_virtualenvs = True +# Don't fail on missing interpreters +nox.options.error_on_missing_interpreters = False + +# Be verbose when running under a CI context +CI_RUN = os.environ.get("CI") +PIP_INSTALL_SILENT = CI_RUN is False +SKIP_REQUIREMENTS_INSTALL = "SKIP_REQUIREMENTS_INSTALL" in os.environ +EXTRA_REQUIREMENTS_INSTALL = os.environ.get("EXTRA_REQUIREMENTS_INSTALL") +SALT_REQUIREMENT = os.environ.get("SALT_REQUIREMENT") or "salt" +COVERAGE_VERSION_REQUIREMENT = "coverage==5.5" + +# Prevent Python from writing bytecode +os.environ["PYTHONDONTWRITEBYTECODE"] = "1" + +# Global Path Definitions +REPO_ROOT = pathlib.Path(__file__).resolve().parent +# Change current directory to REPO_ROOT +os.chdir(str(REPO_ROOT)) + +ARTIFACTS_DIR = REPO_ROOT / "artifacts" +# Make sure the artifacts directory exists +ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) +RUNTESTS_LOGFILE = ARTIFACTS_DIR / "runtests-{}.log".format( + datetime.datetime.now().strftime("%Y%m%d%H%M%S.%f") +) +COVERAGE_REPORT_DB = REPO_ROOT / ".coverage" +COVERAGE_REPORT_PROJECT = ARTIFACTS_DIR.relative_to(REPO_ROOT) / "coverage-project.xml" +COVERAGE_REPORT_TESTS = ARTIFACTS_DIR.relative_to(REPO_ROOT) / "coverage-tests.xml" +JUNIT_REPORT = ARTIFACTS_DIR.relative_to(REPO_ROOT) / "junit-report.xml" + + +def _install_requirements( + session, + *passed_requirements, # pylint: disable=unused-argument + install_coverage_requirements=True, + install_test_requirements=True, + install_source=True, + install_salt=True, + install_extras=None, +): + install_extras = install_extras or [] + if SKIP_REQUIREMENTS_INSTALL is False: + # Always have the wheel package installed + session.install("--progress-bar=off", "wheel", silent=PIP_INSTALL_SILENT) + if install_coverage_requirements: + session.install( + "--progress-bar=off", COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT + ) + # Install the latest salt package + if install_salt: + session.install("--progress-bar=off", SALT_REQUIREMENT, silent=PIP_INSTALL_SILENT) + + if install_test_requirements: + requirements_file = REPO_ROOT / "tests" / "requirements.txt" + install_command = [ + "--progress-bar=off", + "-r", + str(requirements_file.relative_to(REPO_ROOT)), + ] + session.install(*install_command, silent=PIP_INSTALL_SILENT) + + if EXTRA_REQUIREMENTS_INSTALL: + session.log( + "Installing the following extra requirements because the " + "EXTRA_REQUIREMENTS_INSTALL environment variable was set: " + "EXTRA_REQUIREMENTS_INSTALL='%s'", + EXTRA_REQUIREMENTS_INSTALL, + ) + install_command = ["--progress-bar=off"] + install_command += [req.strip() for req in EXTRA_REQUIREMENTS_INSTALL.split()] + session.install(*install_command, silent=PIP_INSTALL_SILENT) + + if install_source: + pkg = "." + if install_extras: + pkg += f"[{','.join(install_extras)}]" + session.install("-e", pkg, silent=PIP_INSTALL_SILENT) + elif install_extras: + pkg = f".[{','.join(install_extras)}]" + session.install(pkg, silent=PIP_INSTALL_SILENT) + + +""" +Nox session to run tests for corresponding python versions and salt versions +""" + + +@nox.session(python=PYTHON_VERSIONS) +@nox.parametrize("api_backend", API_BACKEND) +def tests(session, api_backend): + _install_requirements(session) + + args = [ + "--rootdir", + str(REPO_ROOT), + f"--log-file={RUNTESTS_LOGFILE.relative_to(REPO_ROOT)}", + "--log-file-level=debug", + "--show-capture=no", + f"--junitxml={JUNIT_REPORT}", + "--showlocals", + "-ra", + "-s", + ] + + if session._runner.global_config.forcecolor: + args.append("--color=yes") + if not session.posargs: + args.append("tests/") + else: + for arg in session.posargs: + if arg.startswith("--color") and args[0].startswith("--color"): + args.pop(0) + args.append(arg) + for arg in session.posargs: + if arg.startswith("-"): + continue + if arg.startswith(f"tests{os.sep}"): + break + try: + pathlib.Path(arg).resolve().relative_to(REPO_ROOT / "tests") + break + except ValueError: + continue + else: + args.append("tests/") + try: + session.run( + "coverage", "run", "-m", "pytest", f"--salt-api-backend=rest_{api_backend}", *args + ) + finally: + # Always combine and generate the XML coverage report + try: + session.run("coverage", "combine") + except CommandFailed: + # Sometimes some of the coverage files are corrupt which would + # trigger a CommandFailed exception + pass + + # Generate report for salt code coverage + session.run( + "coverage", + "xml", + "-o", + str(COVERAGE_REPORT_PROJECT), + ) + try: + session.run("coverage", "report", "--show-missing", "--include=pepper/*") + finally: + # Move the coverage DB to artifacts/coverage in order for it to be archived by CI + if COVERAGE_REPORT_DB.exists(): + shutil.move(str(COVERAGE_REPORT_DB), str(ARTIFACTS_DIR / COVERAGE_REPORT_DB.name)) + + +@nox.session(python="3.10") +def flake8(session): + _install_requirements(session) + # Install flake8 + session.install("flake8") + # Run flake8 + session.run("flake8", "tests/", "pepper/", "scripts/pepper", "setup.py") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..db3c13c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[tool.pytest.ini_options] +addopts = "--showlocals --log-file /tmp/pepper-runtests.log --show-capture=no -ra" +testpaths = ["tests"] +norecursedirs = [".git", ".nox"] +usefixtures = ["pepperconfig"] + +[tool.flake8] +max-line-length = 119 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 35df1fa..0000000 --- a/tox.ini +++ /dev/null @@ -1,72 +0,0 @@ -[tox] -envlist = py{3.7,3.8,3.9}-{cherrypy,tornado}-{v3004.2,v3005.1,v3006.0,master},py{3.10}-{cherrypy,tornado}-{v3006.0,master},coverage,flake8 -skip_missing_interpreters = true -skipsdist = false - -[testenv] -passenv = TOXENV, CI, TRAVIS, TRAVIS_*, CODECOV_* -deps = -r{toxinidir}/tests/requirements.txt - v3004.2: salt==3004.2 - v3004.2: jinja2<3.1 - v3005.1: salt==3005.1 - v3006.0: salt==3006.0 - master: git+https://github.com/saltstack/salt.git@master#egg=salt - -changedir = {toxinidir} -setenv = COVERAGE_FILE = {toxworkdir}/.coverage.{envname} -commands = - cherrypy: pytest -v --cov=pepper/ --cov-config=tox.ini --cov-report= {posargs} --salt-api-backend=rest_cherrypy - tornado: pytest -v --cov=pepper/ --cov-config=tox.ini --cov-report= {posargs} --salt-api-backend=rest_tornado - -[testenv:flake8] -deps = - -r {toxinidir}/tests/requirements.txt - flake8 -commands = flake8 tests/ pepper/ scripts/pepper setup.py - -[testenv:coverage] -skip_install = True -deps = - coverage >= 7.0.5, < 8 -setenv = COVERAGE_FILE={toxworkdir}/.coverage -changedir = {toxinidir} -commands = - coverage erase - coverage combine - coverage report -m - coverage html - coverage xml -o {toxworkdir}/coverage.xml - -[testenv:codecov] -deps = codecov -skip_install = True -changedir = {toxinidir} -commands = codecov --file "{toxworkdir}/coverage.xml" - -[testenv:http] -skip_install = True -basepython = python36 -deps = -changedir = {toxinidir}/htmlcov -commands = python -m http.server - -[pytest] -addopts = --showlocals --log-file /tmp/pepper-runtests.log --show-capture=no -ra -testpaths = tests -norecursedirs = .git .tox -usefixtures = pepperconfig - -[flake8] -max-line-length = 119 - -[coverage:run] -branch = true -source = pepper/ -omit = - */.tox/* - */tests/* - */setup.py - -[coverage:report] -skip_covered = True -show_missing = True