diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c982919..a363d80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,23 @@ jobs: - pre-commit uses: ./.github/workflows/test-action.yml + docs: + name: Docs + needs: + - pre-commit + uses: ./.github/workflows/docs-action.yml + + deploy-docs: + name: Deploy Docs + uses: ./.github/workflows/deploy-docs-action.yml + # Only build doc deployments from the main branch of the org repo and never for PRs. + if: >- + github.event_name != 'pull_request' && + github.ref == 'refs/heads/main' + needs: + - docs + - test + set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all @@ -41,29 +58,30 @@ jobs: if: always() needs: - test + - docs steps: - - name: Download Exit Status Files - if: always() - uses: actions/download-artifact@v4 - with: - path: exitstatus - pattern: exitstatus-* - merge-multiple: true + - name: Download Exit Status Files + if: always() + uses: actions/download-artifact@v4 + with: + path: exitstatus + pattern: exitstatus-* + merge-multiple: true - - name: Delete Exit Status Artifacts - if: always() - uses: geekyeggo/delete-artifact@v5 - with: - name: exitstatus-* - useGlob: true - failOnError: false + - name: Delete Exit Status Artifacts + if: always() + uses: geekyeggo/delete-artifact@v5 + with: + name: exitstatus-* + useGlob: true + failOnError: false - - name: Set Pipeline Exit Status - run: | - tree exitstatus - grep -RE 'failure|cancelled' exitstatus/ && exit 1 || exit 0 + - name: Set Pipeline Exit Status + run: | + tree exitstatus + grep -RE 'failure|cancelled' exitstatus/ && exit 1 || exit 0 - - name: Done - if: always() - run: - echo "All workflows finished" + - name: Done + if: always() + run: + echo "All workflows finished" diff --git a/.github/workflows/deploy-docs-action.yml b/.github/workflows/deploy-docs-action.yml new file mode 100644 index 0000000..97f357c --- /dev/null +++ b/.github/workflows/deploy-docs-action.yml @@ -0,0 +1,76 @@ +name: Publish Documentation + +on: + workflow_call: + inputs: + # This is the name of the regular artifact that should + # be transformed into a GitHub Pages deployment. + artifact-name: + type: string + required: false + default: html-docs + +jobs: + + # The released docs are not versioned currently, only the latest ones are deployed. + # + # Versioning support would require either (better): + # * Rebuilding docs for all versions when a new release is made + # * Version selector support in `furo`: https://github.com/pradyunsg/furo/pull/500 + # + # or (more basic): + # * using the `gh-pages` branch and peaceiris/actions-gh-pages + # to be able to deploy to subdirectories. The implementation via + # actions/deploy-pages always replaces the directory root. + + Deploy-Docs-GH-Pages: + name: Publish Docs to GitHub Pages + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + permissions: + pages: write + id-token: write + + runs-on: ubuntu-latest + steps: + - name: Download built docs + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: html-docs + + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + name: html-docs-pages + path: html-docs + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + with: + artifact_name: html-docs-pages + + - name: Delete GitHub Pages artifact + if: always() + uses: geekyeggo/delete-artifact@v5 + with: + name: html-docs-pages + failOnError: false + + - name: Set Exit Status + if: always() + run: | + mkdir exitstatus + echo "${{ job.status }}" > exitstatus/${{ github.job }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v4 + with: + name: exitstatus-${{ github.job }} + path: exitstatus + if-no-files-found: error diff --git a/.github/workflows/docs-action.yml b/.github/workflows/docs-action.yml new file mode 100644 index 0000000..39ccae0 --- /dev/null +++ b/.github/workflows/docs-action.yml @@ -0,0 +1,52 @@ +name: Build Documentation + +on: + workflow_call: + +jobs: + Docs: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 For Nox + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Nox + run: | + python -m pip install --upgrade pip + pip install nox + + - name: Install Doc Requirements + run: | + nox --force-color -e docs --install-only + + - name: Build Docs + env: + SKIP_REQUIREMENTS_INSTALL: true + run: | + nox --force-color -e docs + + - name: Upload built docs as artifact + uses: actions/upload-artifact@v4 + with: + name: html-docs + path: docs/_build/html + + - name: Set Exit Status + if: always() + run: | + mkdir exitstatus + echo "${{ job.status }}" > exitstatus/${{ github.job }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v4 + with: + name: exitstatus-${{ github.job }} + path: exitstatus + if-no-files-found: error diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 73626e4..04d76bf 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,4 +14,6 @@ jobs: uses: ./.github/workflows/ci.yml permissions: contents: write + pages: write + id-token: write pull-requests: read diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..aec3363 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +This project uses [Semantic Versioning](https://semver.org/) - MAJOR.MINOR.PATCH + +# Changelog diff --git a/changelog/.template.jinja b/changelog/.template.jinja new file mode 100644 index 0000000..0cf429a --- /dev/null +++ b/changelog/.template.jinja @@ -0,0 +1,15 @@ +{% if sections[""] %} +{% for category, val in definitions.items() if category in sections[""] %} + +### {{ definitions[category]['name'] }} + +{% for text, values in sections[""][category].items() %} +- {{ text }} {{ values|join(', ') }} +{% endfor %} + +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_ext/saltdomain.py b/docs/_ext/saltdomain.py new file mode 100644 index 0000000..7a85489 --- /dev/null +++ b/docs/_ext/saltdomain.py @@ -0,0 +1,18 @@ +""" +Copied/distilled from Salt doc/_ext/saltdomain.py in order to be able +to use Salt's custom doc refs. +""" + + +def setup(app): + app.add_crossref_type( + directivename="conf_master", + rolename="conf_master", + indextemplate="pair: %s; conf/master", + ) + app.add_crossref_type( + directivename="conf_minion", + rolename="conf_minion", + indextemplate="pair: %s; conf/minion", + ) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/_static/tippy.css b/docs/_static/tippy.css new file mode 100644 index 0000000..6eb9252 --- /dev/null +++ b/docs/_static/tippy.css @@ -0,0 +1,18 @@ +.tippy-box { + background-color: var(--color-background-hover); + color: var(--color-foreground-primary); +} + +body[data-theme="light"] .tippy-box, body[data-theme="auto"] .tippy-box { + box-shadow: 0 0.2rem 0.5rem rgba(0, 0, 0, 0.05), 0 0 0.0625rem rgba(0, 0, 0, 0.1); +} + +body[data-theme="dark"] .tippy-box { + box-shadow: 0 0.2rem 0.5rem rgba(255,255,255, 0.05), 0 0 0.0625rem rgba(255,255,255, 0.1); +} + +@media (prefers-color-scheme: dark) { + body[data-theme="auto"] .tippy-box { + box-shadow: 0 0.2rem 0.5rem rgba(255,255,255, 0.05), 0 0 0.0625rem rgba(255,255,255, 0.1); + } +} diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 0000000..b94feef --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,152 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html +# -- Path setup -------------------------------------------------------------- +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import datetime +import os +import sys +from pathlib import Path + +try: + docs_basepath = os.path.abspath(os.path.dirname(__file__)) +except NameError: + # sphinx-intl and six execute some code which will raise this NameError + # assume we're in the doc/ directory + docs_basepath = os.path.abspath(os.path.dirname(".")) + +PROJECT_ROOT_DIR = Path(docs_basepath).parent + +addtl_paths = ("_ext",) # custom Sphinx extensions + +for addtl_path in addtl_paths: + sys.path.insert(0, os.path.abspath(os.path.join(docs_basepath, addtl_path))) + +# -- Project information ----------------------------------------------------- +this_year = datetime.datetime.today().year +copyright = f"2023 - {this_year}, Salt Extensions organization" + +# Variables to pass into the docs from sitevars.rst for rst substitution +with open("sitevars.rst") as site_vars_file: + site_vars = site_vars_file.read().splitlines() + +rst_prolog = """ +{} +""".format( + "\n".join(site_vars[:]) +) + +# -- General configuration --------------------------------------------------- + +linkcheck_ignore = [r"http://localhost:\d+"] + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.intersphinx", + "sphinxcontrib.spelling", + "sphinx_copybutton", + "sphinxcontrib.towncrier.ext", + "myst_parser", + "sphinx_inline_tabs", + "sphinx_tippy", +] + +myst_enable_extensions = [ + "colon_fence", + "deflist", + "tasklist", + "fieldlist", +] + +tippy_rtd_urls = [ + "https://pytest-salt-factories.readthedocs.io/en/latest/", + "https://docs.pytest.org/en/stable", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + "_build", + "Thumbs.db", + ".DS_Store", + ".vscode", + ".venv", + ".git", + ".gitlab-ci", + ".gitignore", + "sitevars.rst", +] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" +html_title = "Salt Extensions" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] +html_css_files = ["tippy.css"] + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = "" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. Favicons can be up to at least 228x228. PNG +# format is supported as well, not just .ico' +html_favicon = "" + +# ----- Intersphinx Config ----------------------------------------------------------------------------------------> +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "pytest": ("https://docs.pytest.org/en/stable", None), + "salt": ("https://docs.saltproject.io/en/latest", None), + "saltfactories": ("https://pytest-salt-factories.readthedocs.io/en/latest", None), +} +# <---- Intersphinx Config ----------------------------------------------------------------------------------------- + +# Towncrier draft config +towncrier_draft_autoversion_mode = "draft" +towncrier_draft_include_empty = True +towncrier_draft_working_directory = str(PROJECT_ROOT_DIR) + + +def setup(app): + app.add_crossref_type( + directivename="fixture", + rolename="fixture", + indextemplate="pair: %s; fixture", + ) + app.add_crossref_type( + directivename="question", + rolename="question", + indextemplate="pair: %s; question", + ) + app.add_crossref_type( + directivename="path", + rolename="path", + indextemplate="pair: %s; path", + ) + # Allow linking to pytest's confvals. + app.add_object_type( + "confval", + "pytest-confval", + objname="configuration value", + indextemplate="pair: %s; configuration value", + ) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..4e4a99e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,47 @@ +# Create and Maintain Salt Extensions + +Welcome to the documentation for [salt-extension-copier](https://github.com/salt-extensions/salt-extension-copier), a [Copier template](copier-template-ref) designed for creating and maintaining [Salt extensions](saltext-ref). + +This guide covers the entire Salt extension lifecycle, from [creating](creation-target) and [extracting](topics/extraction.md) extensions (as part of the [Great Module Migration](great-migration-ref)) to [updating](update-target), [writing tests](write-tests-target), [writing documentation](writing-docs-target), [publishing docs](docs-publish-target) and [releasing](publishing-target) on PyPI. It also includes information on [submitting](submitting-target) your project to the [`salt-extensions` organization](gh-org-ref) and several other topics. + +That’s a lot to take in, but don’t worry – we’ve got you covered every step of the way! + +```{toctree} +:maxdepth: 2 +:caption: Quickstart +:hidden: + +topics/installation +topics/creation +``` + +```{toctree} +:maxdepth: 2 +:caption: Guides +:hidden: + +topics/testing/index +topics/documenting/index +topics/building +topics/publishing +topics/updating +topics/workflows +topics/organization/index +topics/extraction +``` + +```{toctree} +:maxdepth: 2 +:caption: Reference +:hidden: + +ref/questions +ref/layout +ref/concepts +ref/changelog +``` + +# Indices and tables + +* [](genindex) +* [](search) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..922152e --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/ref/changelog.md b/docs/ref/changelog.md new file mode 100644 index 0000000..8152e33 --- /dev/null +++ b/docs/ref/changelog.md @@ -0,0 +1,12 @@ +# Changelog + +The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +This project uses [Semantic Versioning](https://semver.org/) - MAJOR.MINOR.PATCH + +```{towncrier-draft-entries} +``` + +```{include} ../../CHANGELOG.md +:start-after: '# Changelog' +``` diff --git a/docs/ref/concepts.md b/docs/ref/concepts.md new file mode 100644 index 0000000..0b2e73b --- /dev/null +++ b/docs/ref/concepts.md @@ -0,0 +1,67 @@ +# Concepts + +This section outlines key concepts related to Salt extensions. + +(saltext-ref)= +## Salt extension + +Salt extensions are `pip`-installable Python packages that hook into Salt's module +loading system. They function similarly to custom modules found in `salt://_modules`, `salt://_states` etc., but with additional benefits like a dedicated testing framework and versioned releases. However, they must be installed on each node individually. + +(great-migration-ref)= +## Great module migration + +The Salt Project team has embarked on a significant transformation known as the "Great Module Migration." This initiative involves moving many modules, which were previously part of the core Salt distribution, into separate extensions. While this shift promises to streamline Salt’s core, it also means that many modules will no longer be maintained directly by the Salt team. Instead, their future now depends on community contributors. + +### Timeline and Key Events + +- **February 2016**: Salt extensions [were introduced](https://github.com/saltstack/salt/pull/31218) with the release of Salt 2016.9. +- **November 2020**: A [major update](https://github.com/saltstack/salt/pull/58943) to Salt extensions was released in Salt 3003. [Watch the Salt Extension Overview](https://www.youtube.com/watch?v=hhomJkwxK3Q) video for more details. +- **July 2022**: Tom Hatch submitted a [SEP](https://github.com/saltstack/salt-enhancement-proposals/blob/24660626d9fe26953cd4581be0804ddfd0ceeb90/extention-migration.md) to migrate numerous built-in modules to extensions. +- **October 2023**: A [project board](https://github.com/orgs/salt-extensions/projects/5/views/1) was created to track the migration process. +- **December 2023**: The Broadcom event forced an immediate shift in strategy, leading to the announcement that there [won't be any deprecation period](https://www.youtube.com/watch?v=CubGR8rTy3Y&t=245s) for [community-designated](community-modules-target) modules. +- **January 2024**: A [list of modules to be extracted](https://github.com/saltstack/great-module-migration) was opened for public comment. Take your time to review it. The modules are being migrated into the [salt-extensions](https://github.com/salt-extensions) community org. +- **February 2024**: The [Great Module Purge PR](https://github.com/saltstack/salt/pull/65971) was created. +- **April 2024**: The Great Module Purge PR was merged, making 3008 the target release. + +### Why This Change? + +This drastic change is driven by several key goals: + +1. **Streamline Maintenance**: Reducing the core Salt codebase makes it easier to maintain and improve. +2. **Deprecate Obsolete Modules**: Modules that are no longer relevant or maintained are bulk-deprecated. +3. **Decoupled Releases**: Separating modules from the core Salt release cycle allows for faster updates and better backporting. For extensions that are pure Python, only a single artifact will need to be released, rather than dozens per release. +4. **Efficient Salt Distributions**: Release platform-specific versions of Salt that are smaller and more efficient. +5. **Faster Development**: With fewer modules in the core, testing and review times will decrease, accelerating development. + +### Categories of Modules + +The migration splits modules into three categories: + +1. **Core modules**: These will remain within the Salt codebase and follow the Salt release cycle. +2. **Supported modules (extended core)**: These modules will move to their own repositories within the `saltstack` GitHub organization, where they can be maintained separately. +(community-modules-target)= +3. **Community Modules**: These will be removed from the Salt Core codebase. Community members can take over their maintenance, either in the community-run Salt Extensions GitHub organization or in their own repositories. + +### Why You Should Get Involved + +In conclusion, this migration represents a shift in how Salt is maintained and developed. It opens the door for users and organizations to have a direct impact on the tools they rely on. If you use any of the modules categorized as community modules, their future depends on people like you. By becoming a maintainer or contributor, you can ensure that the modules you depend on continue to thrive and evolve. + +(gh-org-ref)= +## GitHub organization + +The [salt-extensions GitHub organization][saltext-org] was established to offer a community-driven, centralized hub for discovering, creating and maintaining Salt extensions. This organization aims to simplify the development and release process for both new extensions and modules extracted from Salt core starting with the `3008` release. + +Do you care about a set of modules that will be removed from Salt core? +Do you want to publish awesome new Salt functionality? +[Getting involved](org-involve-target) is easy! + +(copier-template-ref)= +## Copier template + +The `salt-extension-copier` template simplifies the creation and maintenance of Salt extensions. Based on user inputs, it generates a custom boilerplate for anyone to develop new Salt functionality and allows for automated updates when new template versions are released. + +Very important: You don't need to publish within the organization to use it – it works for individual projects too! + + +[saltext-org]: https://github.com/salt-extensions diff --git a/docs/ref/layout.md b/docs/ref/layout.md new file mode 100644 index 0000000..270e618 --- /dev/null +++ b/docs/ref/layout.md @@ -0,0 +1,137 @@ +# Project layout +This section provides an overview of the key paths in your Salt extension project. + +:::{path} .copier-answers.yml +::: +## `.copier-answers.yml` +Stores Copier-specific data, including answers to [template questions](questions-target), the template's source URI, and the last version the project was updated to. + +**Do not edit manually.** To change your answers, use `copier update --trust`. To avoid updating the template version, [pass the current version in `vcs-ref`](vcs-ref-target). + +:::{path} CHANGELOG.md +::: +## `CHANGELOG.md` +Contains the project’s changelog. Update this file using [towncrier](changelog-build-target) instead of manually. + +:::{path} README.md +::: +## `README.md` +Provides a brief project overview and developer information. Includes a note about user documentation and, if {question}`docs_url` was set, a link to the hosted documentation. + +:::{path} noxfile.py +::: +## `noxfile.py` +Defines `nox` sessions for [running tests](run-tests-target), [building documentation](build-docs-target), and linting code. + +:::{path} pyproject.toml +::: +## `pyproject.toml` +Holds project metadata, package dependencies and configuration for tools used in the project's lifecycle. + +:::{path} .github +::: +## `.github` +Contains GitHub-related configurations and workflows. This directory is only present if your {question}`source_url` is on GitHub. + +:::{path} .github/workflows +::: +### `.github/workflows` +Houses GitHub Actions {question}`workflows`. + +:::{path} .github/workflows/ci.yml +::: +#### `.github/workflows/ci.yml` +A meta-workflow that triggers other workflows based on inputs. + +:::{important} +Only present when {question}`workflows` == `enhanced`. +::: + +:::{path} .github/workflows/pr.yml +::: +#### `.github/workflows/pr.yml` +Handles workflows for Pull Requests and pushes to the `main` branch. Depending on {question}`workflows`, it either calls centralized workflows in [salt-extensions/central-artifacts](https://github.com/salt-extensions/central-artifacts/tree/main/.github/workflows) or local workflows in {path}`ci.yml <.github/workflows/ci.yml>`. + +:::{path} .github/workflows/tag.yml +::: +#### `.github/workflows/tag.yml` +Triggered by [tag pushes](publishing-target) for tags beginning with `v`. Similar to {path}`pr.yml <.github/workflows/pr.yml>`, it either calls centralized workflows or local ones in {path}`ci.yml <.github/workflows/ci.yml>`. + +:::{path} changelog +::: +## `changelog` +Directory containing [news fragments](news-fragment-target) for `towncrier`. Also includes the default version-specific changelog template in `changelog/.template.jinja`. + +:::{path} docs +::: +## `docs` +Root directory for documentation-related files. + +:::{path} docs/conf.py +::: +### `docs/conf.py` +Contains Sphinx configuration and plugins. + +:::{path} docs/index.rst +::: +### `docs/index.rst` +Homepage for the documentation, (indirectly) linking to all other documentation files. + +:::{hint} +If your project includes a `utils` directory, manually add the corresponding documentation here (not handled by the Copier template or the pre-commit hook). +::: + +:::{path} docs/ref +::: +### `docs/ref` +Directory containing autogenerated module documentation. Typically does not require manual updates, but can be used for custom documents, like a configuration reference. + +:::{path} docs/topics +::: +### `docs/topics` +Intended to hold high-level guides related to your Salt extension, such as `Configuration`. By default, includes an `Installation` guide. + +:::{path} src +::: +## `src` +Root directory for your Salt extension's package. + +:::{path} tests +::: +## `tests` +Root directory for Pytest-based test modules. + +:::{path} tests/conftest.py +::: +### `tests/conftest.py` +Provides default fixtures and basic test setup. + +:::{path} tests/functional +::: +### `tests/functional` +Contains functional tests. + +:::{path} tests/functional/conftest.py +::: +#### `tests/functional/conftest.py` +Provides default fixtures for functional tests. + +:::{path} tests/integration +::: +### `tests/integration` +Contains integration tests. + +:::{path} tests/integration/conftest.py +::: +#### `tests/integration/conftest.py` +Provides default fixtures for integration tests. + +:::{path} tests/unit +::: +### `tests/unit` +Contains unit tests. + +:::{path} tests/unit/conftest.py +::: +#### `tests/unit/conftest.py` +Provides default fixtures for unit tests. diff --git a/docs/ref/questions.md b/docs/ref/questions.md new file mode 100644 index 0000000..d8371cf --- /dev/null +++ b/docs/ref/questions.md @@ -0,0 +1,200 @@ +(questions-target)= +# Template questions + +During project creation or when an update introduces a new variable, you will be prompted with some of the following questions: + +:::{question} project_name +::: +## `project_name` +The name of the project, particularly for PyPI. + +The final name depends on {question}`no_saltext_namespace`. + +**Example**: `vault` + +:::{important} +Do not include the `saltext.` namespace here. +::: + +:::{question} author +::: +## `author` +The main author’s name for PyPI package metadata and licensing. + +**Example**: `Foo Bar` + +:::{question} author_email +::: +## `author_email` +A contact email address for PyPI package metadata. + +**Example**: `foo@b.ar` + +:::{question} integration_name +::: +## `integration_name` +The name of the integrated service, used in autogenerated documentation and README. + +**Example**: `HashiCorp Vault` + +:::{question} summary +::: +## `summary` +A short description of the project for PyPI metadata, documentation, and README. + +**Example**: `Salt extension for interacting with HashiCorp Vault` + +:::{question} url +::: +## `url` +The project's main URL for PyPI metadata, typically the hosted git repository. + +**Example**: `https://github.com/salt-extensions/saltext-vault` + +:::{question} source_url +::: +## `source_url` +The URL of the hosted git repository for PyPI metadata. + +**Example**: `https://github.com/salt-extensions/saltext-vault` + +:::{question} tracker_url +::: +## `tracker_url` +The URL of the issue tracker for PyPI metadata. + +**Example**: `https://github.com/salt-extensions/saltext-vault/issues` + +:::{question} package_name +::: +## `package_name` +The Python package name used for importing. The final name depends on {question}`no_saltext_namespace`. + +**Example**: `vault` + +:::{important} +Do not include the `saltext.` namespace here. +::: + +:::{question} license +::: +## `license` +Select a license. Any license other than Apache 2.0 requires manual management. + +:::{question} license_classifier +::: +## `license_classifier` +The license classifier for PyPI metadata. + +**Example**: `License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)` + +:::{note} +Only asked if a non-Apache {question}`license` is selected. +::: + +:::{question} loaders +::: +## `loaders` +Choose the Salt module types this extension should provide. + +:::{question} salt_version +::: +## `salt_version` +The minimum Salt version to support. Influences package dependencies and versions that are tested against. + +:::{question} python_requires +::: +## `python_requires` +The minimum Python version to support. Also affects pre-commit autoformatting hooks. + +:::{question} max_salt_version +::: +## `max_salt_version` +The maximum Salt version to support. + +:::{hint} +This is only relevant when non-centralized {question}`workflows` are used, as it affects the Salt versions tests are run against. +::: + +:::{question} no_saltext_namespace +::: +## `no_saltext_namespace` +Whether to use the `saltext.` namespace for the Python package. + +:::{question} ssh_fixtures +::: +## `ssh_fixtures` +Include test fixtures for Salt-SSH tests (`salt_ssh_cli` etc.). Defaults to true if `wrapper` {question}`modules ` are included. + +:::{question} test_containers +::: +## `test_containers` +Add support for running containers in the test suite (for functional and integration tests). + +:::{question} workflows +::: +## `workflows` +Select a GitHub Actions workflow style: + +**org** +: Rely on reusable workflows from the `salt-extensions` GitHub organization. + +**enhanced** +: Equivalent workflows to `org`, but stored in the extension repository. + You need to [setup required secrets](required-secrets-target) yourself. + +**basic** +: Provided for compatibility with the deprecated create-salt-extension tool (not recommended for new projects). + +:::{note} +Not asked if {question}`source_url` is not on GitHub. +::: + +:::{question} deploy_docs +::: +## `deploy_docs` +Decide when to [publish documentation to GitHub pages](docs-publish-target): + +**never** +: Don’t publish documentation. + +**release** +: Publish when a release is tagged. + This ensures documentation is in sync with the released functionality. + +**rolling** +: Publish on `push` and `tag` events to the default branch. + This ensures upcoming unreleased changes are displayed in the changelog. + +:::{important} +Ensure your GitHub Actions workflow is [allowed to publish](docs-publish-setup-target) to your GitHub Pages site. +::: + +:::{note} +The current workflows do not support versioned documentation. +::: + +:::{note} +Not asked if {question}`source_url` is not on GitHub or the `basic` {question}`workflows` have been selected. +::: + +:::{question} docs_url +::: +## `docs_url` +The URL for hosted documentation, typically your GitHub Pages URL if using {question}`deploy_docs`. + +**Example**: `https://salt-extensions.github.io/saltext-vault/` + +:::{question} coc_contact +::: +## `coc_contact` +A contact email for Code of Conduct complaints. + +**Example**: `foo@b.ar` + +:::{question} copyright_begin +::: +## `copyright_begin` +The starting year of the copyright range. + +**Example**: `2024` diff --git a/docs/sitevars.rst b/docs/sitevars.rst new file mode 100644 index 0000000..e69de29 diff --git a/docs/topics/building.md b/docs/topics/building.md new file mode 100644 index 0000000..f37cd2e --- /dev/null +++ b/docs/topics/building.md @@ -0,0 +1,26 @@ +(building-target)= +# Building your Saltext + +## Prerequisites + +Ensure your Python environment has the `build` package installed: + +```bash +python -m pip install build +``` + +:::{important} +Your project must be tracked by Git. Make sure you have initialized your repository. +::: + +## Building + +Building your Salt extension's wheel is easy: + +```bash +python -m build --outdir dist/ +``` + +The installable wheel file (`.whl`) can be found in the `dist` directory. You can install it with `salt-pip`/`pip`, provided you don't rename the file. + +For broader distribution, consider [publishing](publishing) your extension to a package index like PyPI. diff --git a/docs/topics/creation.md b/docs/topics/creation.md new file mode 100644 index 0000000..4c68c05 --- /dev/null +++ b/docs/topics/creation.md @@ -0,0 +1,71 @@ +(creation-target)= +# Creation + +With Copier, creating a Salt extension project is easy: + +```bash +copier copy --trust https://github.com/salt-extensions/salt-extension-copier my-awesome-new-saltext +``` + +You are then prompted with questions to configure your project structure. These answers are saved in {path}`.copier-answers.yml` for future [updates](update-target). + +:::{important} + +Copier needs to be invoked with the `--trust` flag in order to enable +custom Jinja extensions (always) and migrations (during updates). +This effectively runs unsandboxed commands on your host, +so ensure you trust the template source! + +* [Jinja extensions][jinja-exts] +* [tasks and migrations][tasks-migrations] +::: + +## Important considerations + +### Organization vs single +Decide early whether to [submit your project](submitting-target) to the [`salt-extensions` GitHub organization](gh-org-ref) or host it in your [own repository](required-secrets-target). This is determined by the `source_url` you provide. + +### GitHub vs other Git host (non-org) +If hosting the repository outside the organization, you can choose your provider freely. Note that the [default workflows](workflows-target) only work on GitHub though. + +(first-steps-target)= +## First steps + +To finalize your project setup, ensure you initialize the Git repository and Python virtual environment and install and run the `pre-commit` hooks. + +### Initialize the repository +```bash +git init -b main +``` + +:::{important} +Some automations assume your default branch is `main`. Ensure this is the case. +::: + +(dev-setup-target)= +### Initialize the Python virtual environment +```bash +python -m venv venv +source venv/bin/activate +python -m pip install -e '.[tests,dev,docs]' +``` + +This creates a virtual environment and installs relevant dependencies, including `nox` and `pre-commit`. + +### Install the `pre-commit` hook +```bash +python -m pre_commit install --install-hooks +``` + +This ensures `pre-commit` runs before each commit. It autoformats and lints your code and ensures the presence of necessary documentation files. To skip these checks temporarily, use `git commit --no-verify`. + +### First commit +```bash +git add . +git commit -m "Initial extension layout" +``` + +In case `pre-commit` modifies or creates files, the commit is aborted. Stage the changes and try again. + +[jinja-exts]: https://github.com/salt-extensions/salt-extension-copier/blob/main/jinja_extensions/saltext.py +[tasks-migrations]: https://github.com/salt-extensions/salt-extension-copier/blob/main/copier.yml diff --git a/docs/topics/documenting/building.md b/docs/topics/documenting/building.md new file mode 100644 index 0000000..49dd90b --- /dev/null +++ b/docs/topics/documenting/building.md @@ -0,0 +1,55 @@ +(build-docs-target)= +# Building documentation + +:::{important} +Ensure `nox` is installed. If you followed the [first steps](first-steps-target), you should be all set. +::: + +## Prerequisites + +On some systems (macOS, WSL, and certain Linux distributions), you must install the `enchant` library to build the documentation. + +:::{tab} Linux/WSL +```bash +sudo apt-get install -y enchant +``` +::: +:::{tab} MacOS +```bash +brew install -y enchant +``` + +:::{important} +On Apple Silicon, you might need to ensure your environment points to the correct library location: + +```bash +export PYENCHANT_LIBRARY_PATH=/opt/homebrew/lib/libenchant-2.2.dylib +``` +::: + +## Build once + +To build your documentation once: + +```bash +nox -e docs +``` + +The rendered documentation will be located in `docs/_build/html`. + +## Live preview +For continuous development, you can start a live preview that automatically reloads when changes are made: + +```bash +nox -e docs-dev +``` + +This command builds the documentation, starts an HTTP server, opens your default browser, and watches for changes. + +:::{note} +If building on a remote system, override the default `localhost` host with: + +```bash +nox -e docs-dev -- --host=1.2.3.4 +``` +::: diff --git a/docs/topics/documenting/changelog.md b/docs/topics/documenting/changelog.md new file mode 100644 index 0000000..5872c85 --- /dev/null +++ b/docs/topics/documenting/changelog.md @@ -0,0 +1,39 @@ +# Keeping a changelog + +Your Saltext project uses [towncrier](https://towncrier.readthedocs.io/en/stable/) to manage and render its {path}`CHANGELOG.md` file, which is included in the rendered documentation as well. + +:::{hint} +If you selected {question}`deploy_docs` == `rolling`, the changelog will display upcoming changes from the `main` branch alongside [crystallized changes from previous releases](changelog-build-target). +::: + +(news-fragment-target)= +## Procedure + +For every user-facing change, ensure your patch includes a corresponding news fragment: + +1. Ensure there is an issue in the bug tracker that describes the context of the change. +2. Before merging a PR, ensure a news fragment describing the change is added to the `changelog` directory. +3. Its file name should follow `..md`, where `resolution` is one of the following: + + * `fixed` + * `added` + * `changed` + * `removed` + * `deprecated` + * `security` + +4. The file contents should be written in Markdown. + +## Example + +Suppose a PR fixes a crash when the `foo.bar` configuration value is missing. The news fragment can be created as follows: + +```bash +echo "Fixed a crash when 'foo.bar' is missing from the configuration" > changelog/23.fixed.md +``` + +Include this file in the PR. + +## Building the changelog + +Before tagging a release, the individual `changelog/*.md` files need to be compiled into the actual changelog. Refer to [Building the changelog](changelog-build-target) for instructions on how to do this. diff --git a/docs/topics/documenting/index.md b/docs/topics/documenting/index.md new file mode 100644 index 0000000..2a1bb78 --- /dev/null +++ b/docs/topics/documenting/index.md @@ -0,0 +1,15 @@ +# Documenting your Saltext + +Your Salt extension's documentation is generated using [sphinx](https://www.sphinx-doc.org/en/master/index.html). + +The environment setup and `sphinx` invocation are managed by [nox](https://nox.thea.codes/en/stable/). + +```{toctree} +:maxdepth: 2 +:hidden: + +building +writing +publishing +changelog +``` diff --git a/docs/topics/documenting/publishing.md b/docs/topics/documenting/publishing.md new file mode 100644 index 0000000..ac2b761 --- /dev/null +++ b/docs/topics/documenting/publishing.md @@ -0,0 +1,29 @@ +(docs-publish-target)= +# Publishing documentation + +If your {question}`source_url` is on GitHub and you selected either `org` or +`enhanced` {question}`workflows`, you can automatically deploy your documentation to your repository's GitHub Pages site. This deployment is controlled by the {question}`deploy_docs` setting. + +(docs-publish-setup-target)= +## Setup + +To enable documentation publishing, follow these steps: + +1. On GitHub, navigate to your repository and click on `Settings`. +2. Select `Pages`. +3. Under `Build and deployment source`, ensure `GitHub Actions` is selected. + +Once configured, your documentation will be automatically published to your GitHub Pages site after a `tag` event or a `push` event (if {question}`deploy_docs` == `rolling`). + +## Further steps + +### Docs URL in metadata + +Ensure the {question}`docs_url` points to your GitHub Pages site. This URL is included in the project's PyPI metadata and ensures that the {path}`README.md` contains a link to the user documentation. + +### Repository website + +Consider setting your GitHub Pages site as the project's website on GitHub: + +1. In your repository, click the settings wheel in the top right corner (not the `Settings` tab in the navigation bar). +2. Under `Website`, select `Use your GitHub Pages website`. diff --git a/docs/topics/documenting/writing.md b/docs/topics/documenting/writing.md new file mode 100644 index 0000000..fddee22 --- /dev/null +++ b/docs/topics/documenting/writing.md @@ -0,0 +1,223 @@ +(writing-docs-target)= +# Writing documentation + +Your project's documentation is located in the {path}`docs` directory. + +## Markup language + +### `docs/*` +You can write dedicated documentation pages using either [MyST](https://myst-parser.readthedocs.io/en/stable/syntax/typography.html) (a Markdown superset) or [reStructuredText (rST)](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html). Mixing files of different formats is allowed. + +### Docstrings +Docstrings in your modules must be written in rST. + +## Tagging changes between versions + +Document any user-facing changes between releases using the `versionadded`, `versionchanged`, `deprecated`, or `versionremoved` directives. + +:::{tab} MyST + changed_arg + : Has been doing something for a long time. + + :::{versionchanged} 1.1.0 + Does it a little differently now. + ::: + + new_arg + : Does something new. + + :::{versionadded} 1.1.0 + ::: +::: + +:::{tab} rST +```rst +changed_arg + Has been doing something for a long time. + + ..versionchanged:: 1.1.0 + Does it a little differently now. + +new_arg + Does something new. + + .. versionadded:: 1.1.0 +``` +::: + +:::{tab} Preview +changed_arg +: Has been doing something for a long time. + + :::{versionchanged} 1.1.0 + Does it a little differently now. + ::: + +new_arg +: Does something new. + + :::{versionadded} 1.1.0 + ::: +::: + +## Cross-references + +### Entities +Improve documentation usability by cross-referencing entities. Some highlights: + +#### Modules +Link to a complete module. + +:::{tab} MyST +```md +{py:mod}`foo ` +``` +::: + +:::{tab} rST +```rst +:py:mod:`foo ` +``` +::: + +:::{hint} +Works for all packages registered in the `docs/conf.py` +`intersphinx_mapping`. This specifically includes Salt core by default. +::: + +#### Functions +Link to a specific function within a module. + +:::{tab} MyST +```md +{py:func}`foo.bar ` +``` +::: + +:::{tab} rST +```rst +:py:func:`foo.bar ` +``` +::: + +:::{hint} +Works for all packages registered in the `docs/conf.py` +`intersphinx_mapping`. This specifically includes Salt core by default. +::: + +#### Salt master configuration +Link to the documentation of a Salt master configuration value. + +:::{tab} MyST +```md +{conf_master}`ssh_minion_opts` +``` +::: + +:::{tab} rST +```rst +:conf_master:`ssh_minion_opts` +``` +::: + +#### Salt minion configuration +Link to the documentation of a Salt minion configuration value. + +:::{tab} MyST +```md +{conf_minion}`order_masters` +``` +::: + +:::{tab} rST +```rst +:conf_minion:`order_masters` +``` +::: + +### Arbitrary anchors +Define and reference custom anchors within the same or different documents. + +:::{tab} MyST +```md +(my-custom-target)= +# Something + +... + +Later in the file, or in another one: + +Please refer to [Something](my-custom-target) +``` +::: + +:::{tab} rST +```rst +.. _my-custom-target: +Something +========= + +... + +Later in the file, or in another one: + +Please refer to :ref:`Something ` +``` +::: + +## Admonitions + +Use admonitions to add notes or emphasize important information. + +:::{tab} MyST + + :::{important} + Ensure you understand the usage of admonitions. + ::: +::: + +:::{tab} rST +```rst +.. important:: + Ensure you understand the usage of admonitions. +``` +::: + +:::{tab} Preview +:::{important} +Ensure you understand the usage of admonitions. +::: + +Common admonitions include: + +* `important` +* `hint` +* `note` +* `warning` +* `attention` +* `tip` + +## Tabs + +You can use tabs to organize content, as shown here. Tabs with the same titles will synchronize. + +:::{tab} MyST + + :::{tab} MyST + ... is a supserset of Markdown. + ::: + + :::{tab} rST + ... is not a supserset of Markdown. + ::: +::: + +:::{tab} rST +```rst +.. tab:: MyST + ... is a supserset of Markdown. + +.. tab:: RST + ... is not a supserset of Markdown. +``` +::: diff --git a/docs/topics/extraction.md b/docs/topics/extraction.md new file mode 100644 index 0000000..1562514 --- /dev/null +++ b/docs/topics/extraction.md @@ -0,0 +1,350 @@ +# Extraction of Salt core modules + +## Scripted example + +A tool named [saltext-migrate](https://github.com/salt-extensions/salt-extension-migrate) was created based on the [manual example](manual-extraction-example) below. It removes many obstacles in the extraction process. Let’s use the same example module (`stalekey`). + +### 1. Install `saltext-migrate` and `git-filter-repo` + +:::{tab} pipx +```bash +pipx install git-filter-repo +pipx install git+https://github.com/salt-extensions/salt-extension-migrate +``` +::: +:::{tab} pip +```bash +pip install git-filter-repo git+https://github.com/salt-extensions/salt-extension-migrate +``` + +If you want to install using `pip`, consider creating a virtual environment beforehand. +::: + +### 2. Run the tool + +:::{important} +Run the tool inside a dedicated directory serving as the working directory for all your Salt extension migrations. This avoids accidental data loss and speeds up repeated migrations. +::: + +```bash +mkdir migrated-saltexts && cd migrated-saltexts +saltext-migrate stalekey +``` + +The tool will: + +1. [Ensure Salt is cloned and the history analysis is available](clone-analyze-target) +2. Filter for paths containing `stalekey` and ask for approval +3. [Filter the history into a separate branch](filter-branch-target), renaming paths as needed +4. Auto-[cleanup the history](clean-history-target), as far as possible non-interactively +5. [Run copier](populate-repo-target) with sane defaults and remove the project starter boilerplate +6. [Create a virtual environment](migration-venv-target) for your project +7. [Apply rewrites](migration-clean-up-target) (with fixes and improvements versus `salt-rewrite`) +8. Install and run pre-commit +9. Provide an overview of issues to fix and next steps + +(manual-extraction-example)= +## A manual module extraction example + +Below are some rough steps to extract an existing set of modules into an extension while preserving the Git history. Let's use the `stalekey` engine as an example. + +### 1. Install the Git history filtering tool + +:::{tab} pipx +```shell +pipx install git-filter-repo +``` +::: +:::{tab} pip +```shell +pip install git-filter-repo +``` + +If you want to install using `pip`, consider creating a virtual environment beforehand. +::: + +(clone-analyze-target)= +### 2. Clone the Salt repo and analyze its history + +```shell +mkdir workdir && cd workdir +git clone https://github.com/saltstack/salt --single-branch +cd salt +git filter-repo --analyze +tree .git/filter-repo/analysis/ +grep stalekey .git/filter-repo/analysis/path-{all,deleted}-sizes.txt | \ + awk '{print $NF}' | sort | uniq | \ + grep -vE '^(.github|doc/ref|debian/|doc/locale|salt/([^/]+/)?__init__.py|tests/(pytests/)?(unit|functional|integration)/conftest.py)' +``` + +The main objective of this step is to find all relevant files (modules, utils, automated tests, fixtures, documentation). For the `stalekey` engine, they are: + +* `salt/engines/stalekey.py` - the engine itself +* `tests/unit/engines/test_stalekey.py` - old style unit tests (historic path, no longer exists in HEAD) +* `tests/pytests/unit/engines/test_stalekey.py` - new-style unit tests using pytest + +(filter-branch-target)= +### 3. Filter the history into a separate branch + +```shell +git checkout -b filter-source +git filter-repo \ + --path salt/engines/stalekey.py \ + --path-rename salt/engines/stalekey.py:src/saltext/stalekey/engines/stalekey.py \ + --path tests/pytests/unit/engines/test_stalekey.py \ + --path-rename tests/pytests/unit/engines/test_stalekey.py:tests/unit/engines/test_stalekey.py \ + --path tests/unit/engines/test_stalekey.py \ + --refs refs/heads/filter-source --force +``` + +The `--path-rename` option moves the files into the directory structure used by Salt extensions. + +(clean-history-target)= +### 4. Clean up the history + +```shell +git log --name-only +git rebase -i --empty=drop --root --committer-date-is-author-date +``` + +The purpose of this step is to drop commits that don’t touch the extracted files plus the last commit that removes them. Merge commits are deleted automatically during the rebase. + +While reviewing the Git log, please note the major contributors (in order to add them as code authors later). + +(populate-repo-target)= +### 5. Populate the extension repo + +Answer the Copier questions, choosing the `engine` module type only, and specify yourself as the author: + +```shell +cd .. +mkdir saltext-stalekey && cd saltext-stalekey +git init --initial-branch=main +copier copy --trust https://github.com/salt-extensions/salt-extension-copier ./ +``` + +Remove unwanted boilerplate files: + +```shell +rm -f tests/**/test_*.py src/**/*_mod.py +``` + +Merge the history: + +```shell +git remote add repo-source ../salt +git fetch repo-source +git merge repo-source/filter-source +git remote rm repo-source +git tag | xargs git tag -d +``` + +(migration-venv-target)= +### 6. Create a virtualenv and activate it + +To create the virtualenv, it is recommended to use the same Python version (MAJOR.MINOR) as the one [listed here](https://github.com/saltstack/salt/blob/master/cicd/shared-gh-workflows-context.yml). + +```shell +python3.10 -m venv venv --prompt saltext-stalekey +source venv/bin/activate +``` + +Please ensure you're inside your virtual environment from here on. + +(migration-clean-up-target)= +### 7. Clean up and test + +Run the automatic fixups: + +```shell +pip install git+https://github.com/saltstack/salt-rewrite +SALTEXT_NAME=stalekey salt-rewrite -F fix_saltext . +``` + +:::{important} +You may need to re-rewrite some imports, as `salt-rewrite` assumes the project is named `saltext.saltext_stalekey` rather than `saltext.stalekey`. +::: + +```shell +pip install -e ".[dev,tests,docs]" +pre-commit install --install-hooks +pre-commit run -a # ensure it is happy +git status +git add . +git commit -m 'Add extension layout' +``` + +Add the main authors to {path}`pyproject.toml`: + +```shell +vi pyproject.toml +git add pyproject.toml +git commit -m 'Add authors' +``` + +Try [running the test suite](run-tests-target) and [building the docs](build-docs-target) locally until both pass, then commit and push it to run the full test suite on GitHub. + +## Basic fixes (automated) +### Unit test module imports + +Unit tests import the modules directly. After migration, these imports +need to be adjusted, otherwise the tests will run against the modules found in Salt, +but still pass (or fail once they are removed in a future release). Example: + +:::{tab} old +```python +from salt.modules import vault +``` +::: + +:::{tab} correct +```python +from saltext.vault.modules import vault +``` +::: + +### Unit test `tests.support` imports + +Many unit tests in the Salt code base use an indirect import for `unittest.mock`. + +:::{tab} old +```python +from tests.support.mock import MagicMock, Mock, patch +``` +::: + +:::{tab} correct +```python +from unittest.mock import MagicMock, Mock, patch +``` +::: + +### Migrated tests in `tests/pytest` + +The generated Salt extension project does not account for a `tests/pytests` subdirectory. Its contents need to be moved to the top-level `tests` directory. + +## Issues needing manual fixing +(utils-dunder-into-saltext-utils)= +### `__utils__` into Salt extension utils + +Some Salt core modules access their utilities via the `__utils__` dunder instead of direct imports, +which ensures that the called utility function has access to Salt's global dunders. + +Accessing a Salt extension's `utils` this way does not work. If this is the case for your extracted set of modules, +you need to adjust the `utils` to not rely on the dunders, e.g. by passing in the required +references: + +:::{tab} old + +```{code-block} python +:caption: salt/modules/foo.py + +def get(entity): + return __utils__["foo.query"](entity) +``` + +```{code-block} python +:caption: salt/utils/foo.py + +def query(entity): + base_url = __opts__.get("foo_base_url", "https://foo.bar") + profile = __salt__["config.option"]("foo_profile") + return __utils__["http.query"](base_url, data=profile) +``` +::: + +:::{tab} correct +```{code-block} python +:caption: saltext/foo/modules/foo.py + +from saltext.foo.utils import foo + + +def get(entity): + base_url = __opts__.get("foo_base_url", "https://foo.bar") + return foo.query(base_url, entity, __salt__["config.option"]) +``` +```{code-block} python +:caption: saltext/foo/utils/foo.py + +import salt.utils.http + + +def query(base_url, entity, config_option): + profile = config_option("foo_profile") + return salt.utils.http.query(base_url, data=profile) +``` +::: + +(utils-dunder-from-saltext-utils)= +### `__utils__` from Salt extension utils +Some modules in `salt.utils` still expect to be accessed via `__utils__`. While this works for modules loaded through the Salt loader (e.g., those using any {question}`loaders`), it fails if your Salt extension’s utils are calling these modules directly. + +Here are some options to address this: + +* Remove the dependency on the core module or call it from the modules calling the utils directly +* Migrate the dependency into your Salt extension repository and modify it locally as described [here](utils-dunder-into-saltext-utils) +* Submit a PR to Salt core with the necessary changes to eliminate the code duplication in the long term. + +(utils-dunder-into-salt-utils)= +### `__utils__` from other Salt extension modules +If any other Saltext module relies on a Salt core utility that requires being called via `__utils__`, it will still work. However, you should consider creating a PR to remove this dependency, as [`__utils__` is scheduled for deprecation](https://github.com/saltstack/salt/issues/62191). + +(pre-pytest-tests)= +### Pre-pytest tests + +Salt core contains both Pytest-based and legacy tests, but Salt extension projects only support `pytest`. To keep legacy tests running, you may need to convert them. If you prefer to skip this task for now, you can: + +- Exclude the corresponding files from `pylint` +- Skip the legacy tests entirely + +```python +# pylint: disable-all +import pytest + +pytest.skip(reason="Old non-pytest tests", allow_module_level=True) +``` + +## Considerations + +(library-dependencies)= +### Library dependencies + +Some modules have library dependencies. Since Salt core cannot include every possible dependency, these modules often include a safeguard to handle missing libraries or library alternatives. They typically use the following pattern: + +```python +HAS_LIBS = False + +try: + import foo + HAS_LIBS = True +except ImportError: + pass + +__virtualname__ = "foobar" + + +def __virtual__(): + if HAS_LIBS: + return __virtualname__ + return False, "Missing 'foo' library" +``` + +If all the dependencies are hard dependencies, declare them in the `dependencies` section of your Saltext's {path}`pyproject.toml` and remove the conditional import logic: + +```python +import foo + +__virtualname__ = "foobar" + + +def __virtual__(): + return __virtualname__ +``` + +For modules that can work with multiple interchangeable libraries, declare at least one of them in the `optional-dependencies` section for `tests` in your {path}`pyproject.toml` to ensure the tests can run. + +(dedicated-docs)= +### Dedicated docs + +Salt core modules often include inline documentation. Consider extracting the general parts of this inline documentation into separate topics within the {path}`docs/topics`directory. diff --git a/docs/topics/installation.md b/docs/topics/installation.md new file mode 100644 index 0000000..bcca0c7 --- /dev/null +++ b/docs/topics/installation.md @@ -0,0 +1,33 @@ +# Installation + +To render the template, you only need a functional [Copier][copier-docs] installation. + +## Copier + +:::{tab} pipx + +It’s recommended to install Copier globally using [pipx][pipx-docs]: + +```bash +pipx install 'copier>=9.1' && \ + pipx inject copier copier-templates-extensions +``` +::: + +:::{tab} pip + +Alternatively, you can install Copier with `pip`, preferably inside a virtual environment: + +```bash +python -m pip install copier copier-templates-extensions +``` +::: + +:::{important} +This template includes custom Jinja extensions, so ensure that [copier-templates-extensions][copier-templates-extensions] is installed in the same environment as `copier`. The example commands above handle this. +::: + +[copier-docs]: https://copier.readthedocs.io/en/stable/ +[copier-multiselect-pr]: https://github.com/copier-org/copier/pull/1386 +[copier-templates-extensions]: https://github.com/copier-org/copier-templates-extensions +[pipx-docs]: https://pipx.pypa.io/stable/ diff --git a/docs/topics/organization/index.md b/docs/topics/organization/index.md new file mode 100644 index 0000000..f643846 --- /dev/null +++ b/docs/topics/organization/index.md @@ -0,0 +1,36 @@ +# The GitHub organization + +## Goals + +The [`salt-extensions` GitHub organization](gh-org-ref) was established in light of the [Great Module Migration](great-migration-ref) to: + +1. Provide users with a central hub for all kinds of Salt modules. +2. Offer developers and contributors a streamlined development experience. +3. Coordinate efforts during the module migration process. + +(org-involve-target)= +## Getting involved + +Your contributions are welcome, whether you're fixing bugs, enhancing extensions, improving documentation, starting a new project, migrating modules from Salt core or providing feedback. + +### Contact + +There are several ways to connect with the community: + +#### Discord +Join the [Salt community Discord][discord-invite] and participate in the `salt-extensions` channel, where many members are active. + +#### Working group +The Salt Extensions working group meets regularly on Microsoft Teams. Check the [Community Calendar](https://saltproject.io/calendar/) or the Discord calendar for upcoming meetings. Meeting notes are available on [GitHub](https://github.com/salt-extensions/community). + +#### GitHub discussions +You can also engage with the community on the organization’s [GitHub discussions](https://github.com/orgs/salt-extensions/discussions) forum. + +[discord-invite]: https://discord.gg/bPah23K7mG + +```{toctree} +:maxdepth: 2 +:hidden: + +submitting +``` diff --git a/docs/topics/organization/submitting.md b/docs/topics/organization/submitting.md new file mode 100644 index 0000000..33dd303 --- /dev/null +++ b/docs/topics/organization/submitting.md @@ -0,0 +1,16 @@ +(submitting-target)= +# Submitting your Saltext + +Have you created a Salt extension that you’d like to host under the Salt Extensions organization? We’d be delighted to welcome your project! + +## Benefits + +By hosting your project within the organization, you gain: + +- A fully configured repository with maintainer rights and pre-configured CI. +- Increased discoverability of your extension. +- Automated Copier template updates (not yet implemented). + +## Requesting a repository + +To request a repository, please complete this [Repository Request Form](https://github.com/salt-extensions/community/issues/new?assignees=&labels=repo&projects=&template=repo.yml&title=%5BRepo+request%5D%3A+) on GitHub. diff --git a/docs/topics/publishing.md b/docs/topics/publishing.md new file mode 100644 index 0000000..73263eb --- /dev/null +++ b/docs/topics/publishing.md @@ -0,0 +1,80 @@ +(publishing-target)= +# Publishing your Saltext + +:::{important} +This guide assumes your repository is hosted on GitHub. + +There are currently no included workflows for other Git hosting providers or CI systems. +::: + +Once your Salt extension is ready, you can submit it to PyPI. + +## 0: Prerequisites + +* Your project is hosted on GitHub. +* It is either in the `salt-extensions` organization or you have set up the [required secrets](required-secrets-target). +* You have commit rights to the repository. +* You have added a git remote `upstream` to your local repository, pointing to the official repository via **SSH**. +* You have followed the [first steps](first-steps-target) to setup your repository and virtual environment. +* You have activated your virtual environment. + +Ensure your `main` branch is up to date: + +```bash +git switch main && git fetch upstream && git rebase upstream/main +``` + +(changelog-build-target)= +## 1: Build the changelog + +Create and switch to a new branch: + +```bash +git switch -c release/100 +``` + +You have been [keeping a changelog](documenting/changelog) with `towncrier`, now is the time to compile it. + +```bash +towncrier build --yes --version v1.0.0 +``` + +This command combines all news fragments into {path}`CHANGELOG.md` and clears them. Commit the change. + +## 2: Submit the changelog + +Submit this commit as a PR and merge it into the default branch on `upstream`. + +:::{tip} +Squash-merging this PR results in a cleaner tag target. +::: + +## 3: Tag a release + +Ensure your `main` branch is up to date (again): + +```bash +git switch main && git fetch upstream && git rebase upstream/main +``` + +Create a new tag named after the version: + +```bash +git tag v1.0.0 +``` + +:::{important} +The tag must start with `v` for the default publishing workflows to work correctly. +::: + +## 4: Push the tag + +Push the new tag upstream to trigger the publishing workflow: + +```bash +git push upstream v1.0.0 +``` + +## 5: Check the result + +If CI passes, a new release should be available on both PyPI and your GitHub repository. diff --git a/docs/topics/testing/index.md b/docs/topics/testing/index.md new file mode 100644 index 0000000..945bdd5 --- /dev/null +++ b/docs/topics/testing/index.md @@ -0,0 +1,13 @@ +# Testing your Saltext + +Your Salt extension's test suite is based on [pytest](https://docs.pytest.org/en/stable/contents.html). + +The environment setup and `pytest` invocation are managed by [nox](https://nox.thea.codes/en/stable/). + +```{toctree} +:maxdepth: 2 +:hidden: + +running +writing +``` diff --git a/docs/topics/testing/running.md b/docs/topics/testing/running.md new file mode 100644 index 0000000..f426418 --- /dev/null +++ b/docs/topics/testing/running.md @@ -0,0 +1,44 @@ +(run-tests-target)= +# Running the test suite + +:::{important} +Ensure `nox` is installed. If you followed the [first steps](first-steps-target), you should be ready to go. +::: + +## Basic + +To run all tests: + +```bash +nox -e tests-3 +``` + +## With parameters + +You can pass `pytest` parameters through `nox` using `--`. + +### Only unit tests + +```bash +nox -e tests-3 -- tests/unit +``` + +### Rerun last failed tests + +```bash +nox -e tests-3 -- --lf +``` + +### Speed up subsequent test runs + +```bash +SKIP_REQUIREMENTS_INSTALL=1 nox -e tests-3 +``` + +### Install extra dependencies + +Useful if you want to invoke a fancier debugger from the tests: + +```bash +EXTRA_REQUIREMENTS_INSTALL="ipdb" PYTHONBREAKPOINT="ipdb.set_trace" nox -e tests-3 +``` diff --git a/docs/topics/testing/writing.md b/docs/topics/testing/writing.md new file mode 100644 index 0000000..c627575 --- /dev/null +++ b/docs/topics/testing/writing.md @@ -0,0 +1,392 @@ +(write-tests-target)= +# Writing tests + +Familiarity with [pytest](https://docs.pytest.org/en/stable/contents.html) is recommended. + +## Overview + +This guide offers a quick overview and best practices specific to Salt extension development. For more details, refer to the [pytest-salt-factories documentation](https://pytest-salt-factories.readthedocs.io/en/latest/). + +## Test types + +There are three main categories of tests: + +**Unit** +: - **Purpose:** Verify low-level or hard-to-reach behavior. Use as a fallback when other test types are too complex to implement. + - **Approach:** Use patching and mocking to isolate the code under test. + - **Example applications:** Exception handling, parsing, utility functions + +**Functional** +: - **Purpose:** Validate that functionality works as expected in a realistic, but lightweight environment (no running Salt daemons). Represents the preferred way of testing, if possible. + - **Approach:** Test modules in a typical environment. Lightweight patching is allowed, but not encouraged. + - **Example applications:** Execution/State/Runner/SDB module tests + +**Integration** +: - **Purpose:** Ensure functionality that depends on running daemons works correctly in a realistic environment. + - **Approach:** Run modules using CLI command wrappers, simulating real-world conditions. + - **Example applications:** Peer publishing, Salt Mine, Salt-SSH (wrapper modules), Reactor + +### Unit tests + +#### Setup and basics +In your test files, you typically import the modules you want to test directly. + +If your module does not reference any Salt-specific global dunders, you can call the function you want to test directly: + +```{code-block} python +:emphasize-lines: 1, 5 + +from saltext.foo.modules import bar + + +def test_bar_baz(): + res = bar.baz() + assert res == "worked" +``` + +However, if your module uses Salt-specific global dunders like `__salt__` or `__opts__`, these dunders won’t be defined yet because the module hasn’t been initialized by the Salt loader. Attempting to call such functions directly would result in a `NameError`. + +To resolve this, define a `configure_loader_modules` fixture. This fixture returns a mapping of modules to initialize to dunder content overrides. + +The overrides can be empty, which just ensures that the dunders are defined: + +```{code-block} python +:emphasize-lines: 2, 8 + +import pytest +from saltext.foo.modules import bar + + +@pytest.fixture +def configure_loader_modules(): + return { + bar: {}, + } +``` + +If you need the `__salt__` dunder to contain specific keys such as `defaults.merge`, and ensure the `defaults` module is properly initialized by the loader, you can define the fixture as follows: + +```{code-block} python +:emphasize-lines: 2-3, 13-21 + +import pytest +from salt.modules import defaults +from saltext.foo.modules import bar + + +@pytest.fixture +def configure_loader_modules(): + opts = { + "value.for.test": True, + } + + return { + bar: { + "__salt__": { + "defaults.merge": defaults.merge, + }, + "__opts__": opts, + }, + defaults: { + "__opts__": opts, + }, + } +``` + +#### Common patterns + +Unit tests usually rely on a subset of the following classes/functions: + +* {py:class}`unittest.mock.Mock` +* {py:class}`unittest.mock.MagicMock` +* {py:func}`unittest.mock.patch` + +Please see the {py:mod}`unittest.mock docs ` for details. + +#### Important fixtures + +##### `minion_opts` +*Scope* +: function + +*Description* +: Provides default `__opts__` for unit tests requiring realistic Salt minion opts. + +##### `master_opts` +*Scope* +: function + +*Description* +: Provides default `__opts__` for unit tests requiring realistic Salt master opts. + + +### Functional tests + +#### Setup and basics + +Functional tests operate within a familiar Salt environment, so you don't need to import the modules you’re testing. The `loaders` fixture provides access to most Salt module types. + +For example, if you're testing an execution module named `foobar`, you can access the initialized module like this: + +```{code-block} python +:emphasize-lines: 5, 7 + +import pytest + + +@pytest.fixture +def foobar_mod(loaders): + # This also works with `states`, `runners` etc. + return loaders.modules.foobar + + +def test_stuff(foobar_mod): + res = foobar_mod.baz() + assert res == "worked" +``` + +If your module requires specific Salt configurations in `__opts__`, you can define configuration overrides using the `minion_config_overrides` or `master_config_overrides` fixtures. These fixtures are scoped to the module, meaning they apply to all tests in the same file: + +```python +import pytest + + +@pytest.fixture(scope="module") +def minion_config_overrides(): + return { + "my_conf": "val", + } +``` + +#### Common patterns + +##### Creating temporary files + +You can create temporary files using {py:func}`pytest.helpers.temp_file `, preferrably as a context manager: + +```{code-block} python +:emphasize-lines: 13 + +import pytest +from textwrap import dedent + + +def test_stuff(tmp_path, loaders, minion_opts): + file_name = "foo" + file_contents = dedent( + """ + {{ opts | json }} + """ + ).strip() + + with pytest.helpers.temp_file(file_name, file_contents, tmp_path) as test_file: + res = loaders.modules.slsutil.renderer(str(test_file)) + assert res == minion_opts +``` + +In this example, a temporary file is created, used, and cleaned up automatically within the test. + +:::{tip} +Temporary files are often created within fixtures, not the tests themselves. This separation of concerns improves code reuse and ensures that the actual tests are concise. +::: + +##### Testing state modules + +###### Return value assertions + +When calling state modules in a functional test, the return value is a wrapper around the standard dictionary return. You should access its properties using the following pattern: + +```{code-block} python +:emphasize-lines: 3-5 + +def test_state_module(states): + ret = states.my_state.present("foo") + assert ret.result is True + assert "as specified" in ret.comment + assert not ret.changes +``` + +###### Test mode + +State modules can also be called with `test=True` during functional tests: + +```{code-block} python +:emphasize-lines: 2 + +def test_state_module_test(states): + ret = states.my_state.present("foo", test=True) + assert ret.result is None + assert "would have" in ret.comment + assert ret.changes +``` + +#### Important fixtures + +##### `loaders` +*Scope* +: function + +*Description* +: An instance of {py:class}`Loaders `, provides access to Salt loaders for several module types via its attributes. + +*Example* +: ```python + loaders.modules.test.ping() + ``` +##### `modules` +*Scope* +: function + +*Description* +: Shortcut for `loaders.modules`. + +##### `states` +*Scope* +: function + +*Description* +: Shortcut for `loaders.states`. + + +### Integration tests + +#### Setup and basics + +Integration tests run within a familiar Salt environment, hence you don't need to import the modules you're testing. Instead, you can run your modules using specific fixtures that wrap familiar CLI commands. + +These fixtures invoke a subprocess, so their return value is a wrapper around the command's result: + +```{code-block} python +:emphasize-lines: 4, 7 + +def test_stuff(salt_call_cli): + res = salt_call_cli.run("foobar.baz") + # Ensure the execution did not error. + assert res.returncode == 0 + # The actual return value is stored in the `data` attribute. + # It is automatically hydrated into the appropriate Python type. + assert res.data == {"worked": True} +``` + +If your modules require specific Salt configurations, you can override the Salt master or minion configuration in your project's `tests/conftest.py` by defining a fixture named `master_config` or `minion_config`: + +```python +import pytest + + +@pytest.fixture(scope="package") +def master_config(): + return { + "ext_pillar": [ + {"my_pillar": {}}, + ], + } +``` + +#### Common patterns + +##### Creating temporary state files + +To test specific modules within the context of the state machinery, you can create a temporary state file in the Salt master's `file_roots`: + +```{code-block} python +:emphasize-lines: 14 + +from textwrap import dedent + + +def test_foobar_in_state_apply(salt_call_cli, master): + sls = "foobar_test" + file_contents = dedent( + """ + Test this: + foobar.present: + - name: baz + """ + ) + + with master.state_tree.base.temp_file(f"{sls}.sls", file_contents): + res = salt_call_cli.run("state.apply", sls) + assert res.returncode == 0 +``` + +#### Important fixtures + +##### `salt_call_cli` +*Scope* +: function + +*Description* +: Runs `salt-call` commands, typically used in most integration tests. + +*Example* +: ```python + res = salt_call_cli.run("state.highstate") + assert res.returncode == 0 + ``` + +##### `salt_run_cli` +*Scope* +: function + +*Description* +: Runs `salt-run` commands, often used in runner integration tests or for setting up master fixtures (e.g. syncing the fileserver). + +*Example* +: ```python + res = salt_run_cli.run("fileserver.update") + assert res.returncode == 0 + assert res.data is True + ``` + +##### `salt_ssh_cli` +*Scope* +: module + +*Description* +: Runs `salt-ssh` commands, usually for `wrapper` module tests. Available when the extension has enabled for `wrapper` {question}`loaders` or {question}`ssh_fixtures`. + +*Example* +: ```python + res = salt_ssh_cli.run("foobar.baz") + assert res.returncode == 0 + assert res.data == {"worked": True} + ``` + +##### `master` +*Scope* +: package + +*Description* +: Provides an instance of {py:class}`saltfactories.daemons.master.SaltMaster`. Example uses include inspecting the current master configuration or creating temporary files in the state/pillar tree. + +*Example* +: Temporary state file in `base` env + + ```{parsed-literal} + with {py:class}`master `.{py:class}`state_tree `.{py:class}`base `.{py:meth}`temp_file `("file_name", "contents") as temp_sls: + ``` +: Temporary pillar file in `prod` env + + ```{parsed-literal} + with {py:class}`master `.{py:class}`pillar_tree `.{py:class}`prod `.{py:meth}`temp_file `("file_name", "contents") as temp_pillar: + ``` + + +##### `minion` +*Scope* +: package + +*Description* +: Provides an instance of {py:class}`saltfactories.daemons.minion.SaltMinion`. Example uses include inspecting the current minion configuration or creating temporary files in the state/pillar tree when `file_client` is set to `local`. + +: Temporary state file in `prod` env + + ```{parsed-literal} + with {py:class}`minion `.{py:class}`state_tree `.{py:class}`prod `.{py:meth}`temp_file `("file_name", "contents") as temp_sls: + ``` +: Temporary pillar file in `base` env + + ```{parsed-literal} + with {py:class}`minion `.{py:class}`pillar_tree `.{py:class}`base `.{py:meth}`temp_file `("file_name", "contents") as temp_pillar: + ``` diff --git a/docs/topics/updating.md b/docs/topics/updating.md new file mode 100644 index 0000000..49dc315 --- /dev/null +++ b/docs/topics/updating.md @@ -0,0 +1,79 @@ +(update-target)= +# Updating your Saltext + +Copier allows you to keep your Salt extension repository up to date with the latest best practices. Updating fetches the latest template release, uses your saved answers as defaults, and applies your customizations. + +## Manual update + +Manual updates let you: + +* Update answers to existing questions +* Provide answers to new questions +* Resolve merge conflicts when customized files change in the template + +### Workflow + +1. Ensure you are in your project's root directory and run: + ```bash + git switch main && git switch -c copier-update + copier update --trust + ``` +2. Review your previous answers and provide answers to any new questions. +3. Check for and resolve any merge conflicts: + ```bash + git status + ``` +4. Remove any unwanted regenerated boilerplate files (a subset of `ls src/**/*_mod.py tests/**/test_*.py`).[^skip-if-exists-issue] +5. Review, then stage the changes: + ```bash + git status + git diff + git add . + ``` +6. Run `pre-commit` on the entire repository and ensure it passes: + ```bash + pre-commit run -a + ``` +7. Commit and submit the update via a PR: + ```bash + git add . && git commit -m "Update to Copier template 0.3.7" && git push + ``` + +[^skip-if-exists-issue]: Currently, there is an issue where default boilerplate files are regenerated if they were deleted. This will be fixed soon, but depends on a new Copier release. + +#### Skip reviewing answers + +To skip reviewing existing answers: + +```bash +copier update --trust --skip-answered +``` + +#### Always use default answers + +To skip new questions and use defaults: + +```bash +copier update --trust --skip-answered --defaults +``` + +:::{hint} +This command is non-interactive. +::: + +(vcs-ref-target)= +#### Update to a specific version + +To update to a specific template version: + +```bash +copier update --trust --skip-answered --vcs-ref=0.3.7 +``` + +## Automatic update + +[RenovateBot](https://docs.renovatebot.com/) supports updating Copier templates. If your repository is hosted within the `salt-extensions` GitHub organization, automated PRs for template updates will be provided in the future. Review these carefully before merging. + +:::{important} +This feature is not yet implemented due to an outstanding issue in Copier. +::: diff --git a/docs/topics/workflows.md b/docs/topics/workflows.md new file mode 100644 index 0000000..3611f8c --- /dev/null +++ b/docs/topics/workflows.md @@ -0,0 +1,55 @@ +(workflows-target)= +# Workflows + +Your Salt extension repository includes several workflows out of the box if your {question}`source_url` is on GitHub. + +:::{note} +The workflows used within the `salt-extensions` organization (`org`) are equivalent +to the `enhanced` ones. +::: + +## Provided functions + +The workflows currently: + +* Ensure `pre-commit` checks pass +* Run the test suite +* Build the documentation +* Optionally deploy built documentation to GitHub Pages +* Optionally build and release your project to PyPI + +## Repository setup + +### Required settings (all) +If publishing documentation to GitHub Pages, ensure you have +[set up your repository to allow deployments from GitHub Actions](docs-publish-setup-target). + +(required-secrets-target)= +### Required secrets (non-org) +If your repository is not hosted within the `salt-extensions` organization, you need to add the following secrets: + +`PYPI_API_TOKEN` +: An [API token for PyPI](https://pypi.org/help/#apitoken) for [releasing your Saltext](publishing-target). + +`TEST_PYPI_API_TOKEN` +: An [API token for TestPyPI](https://test.pypi.org/help/#apitoken) for testing the [release of your Saltext](publishing-target). + +:::{important} +Workflows are expected to migrate to [Trusted Publishing](https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/) soon, making these secrets obsolete. +::: + +## Important artifacts + +After a workflow run, several artifacts are available for download +on the action summary page (scroll down). + +### `runtests-*.log` +Contains logs generated during a specific test run, useful for debugging test failures. + +### `html-docs` +The built HTML documentation, also available for preview when triggered by a Pull Request. + +## Workflows call stack +1. {path}`.github/workflows/pr.yml` or {path}`.github/workflows/tag.yml` is triggered +2. {path}`.github/workflows/ci.yml` (or its equivalent centralized workflow) is called as the main entry point to CI +3. Depending on the event and inputs, select additional workflows perform the necessary tasks. diff --git a/noxfile.py b/noxfile.py index 1682014..52fbaac 100755 --- a/noxfile.py +++ b/noxfile.py @@ -1,5 +1,6 @@ # pylint: disable=missing-module-docstring,import-error,protected-access,missing-function-docstring import datetime +import json import os import pathlib import shutil @@ -44,6 +45,20 @@ DEV_REQUIREMENTS = ("pylint",) +DOCS_REQUIREMENTS = ( + "sphinx", + "sphinxcontrib-spelling", + "sphinx-copybutton", + "myst_parser", + "furo", + "sphinx-inline-tabs", + "towncrier==22.12.0", + "sphinxcontrib-towncrier", + "sphinx_tippy", +) + +DOCSAUTO_REQUIREMENTS = ("sphinx-autobuild",) + TESTS_REQUIREMENTS = ( "copier>=9.1", "copier-templates-extensions", @@ -258,3 +273,107 @@ def lint_tests_pre_commit(session): else: paths = ["tests/"] _lint_pre_commit(session, ".pylintrc", flags, paths) + + +@nox.session +def docs(session): + """ + Build Docs + """ + _install_requirements( + session, + *DOCS_REQUIREMENTS, + ) + os.chdir("docs/") + session.run("make", "clean", external=True) + session.run("make", "linkcheck", "SPHINXOPTS=-W", external=True) + session.run("make", "html", "SPHINXOPTS=-W", external=True) + os.chdir(str(REPO_ROOT)) + + +@nox.session(name="docs-html") +@nox.parametrize("clean", [False, True]) +@nox.parametrize("include_api_docs", [False, True]) +def docs_html(session, clean, include_api_docs): + """ + Build Sphinx HTML Documentation + + TODO: Add option for `make linkcheck` and `make coverage` + calls via Sphinx. Ran into problems with two when + using Furo theme and latest Sphinx. + """ + _install_requirements( + session, + *DOCS_REQUIREMENTS, + ) + build_dir = pathlib.Path("docs", "_build", "html") + sphinxopts = "-Wn" + if clean: + sphinxopts += "E" + args = [sphinxopts, "--keep-going", "docs", str(build_dir)] + session.run("sphinx-build", *args, external=True) + + +@nox.session(name="docs-dev") +def docs_dev(session) -> None: + """ + Build and serve the Sphinx HTML documentation, with live reloading on file changes, via sphinx-autobuild. + + Note: Only use this in INTERACTIVE DEVELOPMENT MODE. This SHOULD NOT be called + in CI/CD pipelines, as it will hang. + """ + _install_requirements( + session, + *DOCS_REQUIREMENTS, + *DOCSAUTO_REQUIREMENTS, + ) + + build_dir = pathlib.Path("docs", "_build", "html") + + # Allow specifying sphinx-autobuild options, like --host. + args = ["--watch", "."] + session.posargs + if not any(arg.startswith("--host") for arg in args): + # If the user is overriding the host to something other than localhost, + # it's likely they are rendering on a remote/headless system and don't + # want the browser to open. + args.append("--open-browser") + args += ["docs", str(build_dir)] + + if build_dir.exists(): + shutil.rmtree(build_dir) + + session.run("sphinx-autobuild", *args) + + +@nox.session(name="docs-crosslink-info") +def docs_crosslink_info(session): + """ + Report intersphinx cross links information + """ + _install_requirements( + session, + install_extras=["docs"], + ) + os.chdir("docs/") + intersphinx_mapping = json.loads( + session.run( + "python", + "-c", + "import json; import conf; print(json.dumps(conf.intersphinx_mapping))", + silent=True, + log=False, + ) + ) + intersphinx_mapping_list = ", ".join(list(intersphinx_mapping)) + try: + mapping_entry = intersphinx_mapping[session.posargs[0]] + except IndexError: + session.error( + f"You need to pass at least one argument whose value must be one of: {intersphinx_mapping_list}" + ) + except KeyError: + session.error(f"Only acceptable values for first argument are: {intersphinx_mapping_list}") + session.run( + "python", "-m", "sphinx.ext.intersphinx", mapping_entry[0].rstrip("/") + "/objects.inv" + ) + os.chdir(str(REPO_ROOT)) diff --git a/project/README.md.j2 b/project/README.md.j2 index 2140267..de9f763 100644 --- a/project/README.md.j2 +++ b/project/README.md.j2 @@ -4,13 +4,12 @@ ## Security -If you think you have found a security vulnerability, see -[Salt's security guide][security]. +If you discover a security vulnerability, please refer +to [Salt's security guide][security]. ## User Documentation -This README is for people aiming to contribute to the project. -If you just want to get started with the extension, check out the +For setup and usage instructions, please refer to the {%- if docs_url %} [User Documentation][docs]. {%- else %} @@ -19,73 +18,81 @@ module docstrings (for now, documentation is coming!). ## Contributing -The {{ project_name_full }} project team welcomes contributions from the community. +The {{ project_name_full }} project welcomes contributions from anyone! -The [Salt Contributing guide][salt-contributing] has a lot of relevant -information, but if you'd like to jump right in here's how to get started: +The [Salt Extensions guide][salt-extensions-guide] provides comprehensive instructions on all aspects +of Salt extension development, including [writing tests][writing-tests], [running tests][running-tests], +[writing documentation][writing-docs] and [rendering the docs][rendering-docs]. +### Quickstart + +To get started contributing, first clone this repository (or your fork): ```bash # Clone the repo -git clone --origin salt {{ source_url | replace("https://", "git@") | replace("/", ":", 1) }}.git +git clone --origin upstream {{ source_url | replace("https://", "git@") | replace("/", ":", 1) }}.git # Change to the repo dir cd {{ project_name_full }} +``` -# Create a new venv -python3 -m venv env --prompt {{ project_name_full }} -source env/bin/activate +Then follow the [first steps][first-steps], skipping the repository initialization and first commit. -# On mac, you may need to upgrade pip -python -m pip install --upgrade pip +{%- if "github.com" in source_url %} -# On WSL or some flavors of linux you may need to install the `enchant` -# library in order to build the docs -sudo apt-get install -y enchant +### Pull request -# Install extension + test/dev/doc dependencies into your environment -python -m pip install -e '.[tests,dev,docs]' +Always make changes in a feature branch: -# Run tests! -python -m nox -e tests-3 +```bash +git switch -c my-feature-branch +``` -# skip requirements install for next time -export SKIP_REQUIREMENTS_INSTALL=1 +To [submit a Pull Request][submitting-pr], you'll need a fork of this repository in +your own GitHub account. If you followed the instructions above, +set your fork as the `origin` remote now: -# Build the docs, serve, and view in your web browser: -python -m nox -e docs && (cd docs/_build/html; python -m webbrowser localhost:8000; python -m http.server; cd -) +```bash +git remote add origin git@github.com:.git ``` -Writing code isn't the only way to contribute! We value contributions in any of -these areas: +Ensure you followed the [first steps][first-steps] and commit your changes, fixing any +failing `pre-commit` hooks. Then push the feature branch to your fork and submit a PR. -* Documentation - especially examples of how to use this module to solve - specific problems. -* Triaging [issues][issues]{%- if "github.com" in source_url %} and participating in [discussions][discussions]{%- endif %} +{%- endif %} + +### Ways to contribute + +Contributions come in many forms, and they’re all valuable! Here are some ways you can help +without writing code: + +* **Documentation**: Especially examples showing how to use this project + to solve specific problems. +* **Triaging issues**: Help manage [issues][issues]{%- if "github.com" in source_url %} and participate in [discussions][discussions]{%- endif %}. {%- if "github.com" in source_url %} -* Reviewing [Pull Requests][PRs] (we really like - [Conventional Comments][comments]!) +* **Reviewing [Pull Requests][PRs]**: We especially appreciate reviews using [Conventional Comments][comments]. {%- endif %} -You could also contribute in other ways: +You can also contribute by: * Writing blog posts -* Posting on social media about how you used Salt + {{ integration_name or project_name.replace("-", " ").title() }} to solve your - problems, including videos +* Sharing your experiences using Salt + {{ integration_name or project_name.replace("-", " ").title() }} + on social media * Giving talks at conferences * Publishing videos -* Asking/answering questions in IRC, Discord or email groups +* Engaging in IRC, Discord or email groups Any of these things are super valuable to our community, and we sincerely appreciate every contribution! - -For more information, build the docs and head over to http://localhost:8000/ — -that's where you'll find the rest of the documentation. - - [security]: https://github.com/saltstack/salt/blob/master/SECURITY.md -[salt-contributing]: https://docs.saltproject.io/en/master/topics/development/contributing.html +[salt-extensions-guide]: https://salt-extensions.github.io/salt-extension-copier/ +[writing-tests]: https://salt-extensions.github.io/salt-extension-copier/topics/testing/writing.html +[running-tests]: https://salt-extensions.github.io/salt-extension-copier/topics/testing/running.html +[writing-docs]: https://salt-extensions.github.io/salt-extension-copier/topics/documenting/writing.html +[rendering-docs]: https://salt-extensions.github.io/salt-extension-copier/topics/documenting/building.html +[first-steps]: https://salt-extensions.github.io/salt-extension-copier/topics/creation.html#initialize-the-python-virtual-environment +[submitting-pr]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork {%- if tracker_url %} [issues]: {{ tracker_url }} {%- endif %} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3918e42 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[tool.build_sphinx] +source_dir = "docs" +build_dir = "build/sphinx" + +[tool.black] +line-length = 100 + +[tool.isort] +force_single_line = true +profile = "black" +line_length = 100 + +[tool.towncrier] +filename = "CHANGELOG.md" +template = "changelog/.template.jinja" +directory = "changelog/" +start_string = "# Changelog\n" +underlines = ["", "", ""] +title_format = "## {version} ({project_date})" +issue_format = "[#{issue}](https://github.com/salt-extensions/salt-extension/copier/issues/{issue})" + +[[tool.towncrier.type]] +directory = "removed" +name = "Removed" +showcontent = true + +[[tool.towncrier.type]] +directory = "deprecated" +name = "Deprecated" +showcontent = true + +[[tool.towncrier.type]] +directory = "changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory = "fixed" +name = "Fixed" +showcontent = true + +[[tool.towncrier.type]] +directory = "added" +name = "Added" +showcontent = true + +[[tool.towncrier.type]] +directory = "security" +name = "Security" +showcontent = true