From 9ec80005d5b36f6ad591e6099263589e6b827670 Mon Sep 17 00:00:00 2001 From: Matthew Watkins Date: Mon, 3 Jun 2024 18:38:57 +0100 Subject: [PATCH] Chore: Update all GHA workflows manually [skip ci] Signed-off-by: Matthew Watkins --- .flake8 | 3 + .github/workflows/bootstrap.yaml | 338 +++++++++++++++++++++------ .github/workflows/builds.yaml | 47 +++- .github/workflows/documentation.yaml | 18 +- .github/workflows/notebooks.yaml | 84 +++++++ .github/workflows/release.yaml | 29 ++- .github/workflows/security.yaml | 13 +- .github/workflows/test-release.yaml | 21 +- .github/workflows/testing.yaml | 15 +- .pre-commit-config.yaml | 124 +++++----- 10 files changed, 507 insertions(+), 185 deletions(-) create mode 100644 .flake8 create mode 100644 .github/workflows/notebooks.yaml diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..cb23f32 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 160 +extend-ignore = E203, E501 diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index a1ebb30..3e1df48 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -12,7 +12,7 @@ jobs: name: "Update DevOps tooling" runs-on: ubuntu-latest permissions: - # IMPORTANT: mandatory to update content/actions/PRs + # IMPORTANT: mandatory to create or update content/actions/pr contents: write actions: write pull-requests: write @@ -20,11 +20,8 @@ jobs: steps: - name: "Checkout primary repository" uses: actions/checkout@v4 - with: - # Note: Requires a specific/defined Personal Access Token - token: ${{ secrets.ACTIONS_WORKFLOW }} - - name: "Pull workflows from central repository" + - name: "Pull devops content from repository" uses: actions/checkout@v4 with: repository: "os-climate/devops-toolkit" @@ -36,115 +33,308 @@ jobs: GH_TOKEN: ${{ github.token }} # yamllint disable rule:line-length run: | - ### SHELL CODE START ### + #SHELLCODESTART + set -euo pipefail + # set -x + + # Define variables + + DEVOPS_DIR=".devops" + AUTOMATION_BRANCH="update-devops-tooling" REPO_DIR=$(git rev-parse --show-toplevel) - # Ensure working from top-level of GIT repository - CURRENT_DIR=$(pwd) - if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then - echo "Changing directory to: $REPO_DIR" - if ! (cd "$REPO_DIR"); then - echo "Error: unable to change directory"; exit 1 - fi - fi + GIT_ORIGIN=$(git config --get remote.origin.url) + REPO_NAME=$(basename -s .git "$GIT_ORIGIN") + EXCLUDE_FILE=".devops-exclusions" + DEVOPS_REPO='git@github.com:os-climate/devops-toolkit.git' + HEAD_BRANCH=$(git rev-parse --abbrev-ref HEAD) + + # Content folder defines the files and folders to update + FILES="$DEVOPS_DIR/content/files.txt" + FOLDERS="$DEVOPS_DIR/content/folders.txt" - # Define a function to allow selective opt-out of devops tooling - OPT_OUT=".devops-exclusions" + # Define functions + + perform_folder_operation() { + FS_PATH="$1" + if [ -d "$DEVOPS_DIR"/"$FS_PATH" ]; then + echo "Scanning target folder content at: $FS_PATH" + return 0 + else + echo "Upstream folder NOT found: $FS_PATH [skipping]" + return 1 + fi + } + + # Allows for selective opt-out components on a per-path basis perform_operation() { - ELEMENT="$1" - if [ ! -f "$OPT_OUT" ]; then - # Opt-out file does not exist; all operations will be performed + FS_PATH="$1" + if [ ! -f "$DEVOPS_DIR"/"$FS_PATH" ]; then + echo "Skipping missing upstream file at: $FS_PATH" + return 1 + fi + # Elements excluded from processing return exit status 1 + if [ ! -f "$EXCLUDE_FILE" ]; then + return 0 + elif [ "$FS_PATH" = "$EXCLUDE_FILE" ]; then + # The exclusion file itself is never updated by automation + return 1 + elif (grep -Fxq "$FS_PATH" "$EXCLUDE_FILE" > /dev/null); then + # Element listed; exclude from processing return 1 else - if grep -Fxq "$ELEMENT" "$OPT_OUT" - then - # Element is excluded from processing + # Element not found in exclusion file; process it + return 0 + fi + } + + # Only updates file if it has changed + selective_file_copy() { + # Receives a single file path as argument + # SHA_SRC=$(sha1sum "$DEVOPS_DIR/$1" | awk '{print $1}') + # SHA_DST=$(sha1sum "$1" 2>/dev/null | awk '{print $1}' || :) + # if [ "$SHA_SRC" != "$SHA_DST" ]; then + if ! (cmp "$DEVOPS_DIR/$1" "$1"); then + echo "Copying: $1" + cp "$DEVOPS_DIR/$1" "$1" + git add "$1" + fi + } + + check_pr_for_author() { + AUTHOR="$1" + printf "Checking for pull requests by: %s" "$AUTHOR" + # Capture the existing PR number + PR_NUM=$(gh pr list --state open -L 1 \ + --author "$AUTHOR" --json number | \ + grep "number" | sed "s/:/ /g" | awk '{print $2}' | \ + sed "s/}//g" | sed "s/]//g") + if [ -z "$PR_NUM" ]; then + echo " [none]" + return 1 + else + echo " [$PR_NUM]" + echo "Running: gh pr checkout $PR_NUM" + if (gh pr checkout "$PR_NUM"); then return 0 else - # Element should be processed - return 1 + echo "Failed to checkout GitHub pull request" + echo "Check errors/output for the cause" + return 2 fi fi } - echo "Removing remote branch if it exists: update-devops-tooling" - git push origin --delete update-devops-tooling || : - STRING=$(dd if=/dev/urandom bs=1k count=1 2>/dev/null | tr -dc 'a-zA-Z0-9' | head -c 10) - git checkout -b "update-$STRING" + check_prs() { + # Define users to check for pre-existing pull requests + AUTOMATION_USER="github-actions[bot]" + if [[ -n ${GH_TOKEN+x} ]]; then + GITHUB_USERS="$AUTOMATION_USER" + else + GITHUB_USERS=$(gh api user | jq -r '.login') + # Check local user account first, if enumerated + GITHUB_USERS+=" $AUTOMATION_USER" + fi + + # Check for existing pull requests opened by this automation + for USER in $GITHUB_USERS; do + if (check_pr_for_author "$USER"); then + return 0 + else + STATUS="$?" + fi + if [ "$STATUS" -eq 1 ]; then + continue + elif [ "$STATUS" -eq 2 ]; then + echo "Failed to checkout pull request"; exit 1 + fi + done + return 1 + } + + # Check if script is running in GHA workflow + in_github() { + if [ -z ${GITHUB_RUN_ID+x} ]; then + echo "Script is NOT running in GitHub" + return 1 + else + echo "Script is running in GitHub" + return 0 + fi + } - # Configure GIT - TEST=$(git config -l) + # Check if user is logged into GitHub + logged_in_github() { + if (gh auth status); then + echo "Logged in and authenticated to GitHb" + return 0 + else + echo "Not logged into GitHub, some script operations unavailable" + return 1 + fi + } + + # Main script entry point + + echo "Repository name and HEAD branch: $REPO_NAME [$HEAD_BRANCH]" + + # Ensure working from top-level of GIT repository + CURRENT_DIR=$(pwd) + if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then + echo "Changing directory to: $REPO_DIR" + if ! (cd "$REPO_DIR"); then + echo "Error: unable to change directory"; exit 1 + fi + fi + + # Stashing only used during development/testing + # Check if there are unstaged changes + # if ! (git diff --exit-code --quiet); then + # echo "Stashing unstaged changes in current repository" + # git stash -q + # fi + + # Configure GIT environment only if NOT already configured + # i.e. when running in a GitHub Actions workflow + TEST=$(git config -l > /dev/null 2>&1) if [ -n "$TEST" ]; then git config user.name "github-actions[bot]" git config user.email \ "41898282+github-actions[bot]@users.noreply.github.com" fi - FOLDERS=".github .github/workflows scripts" - for FOLDER in ${FOLDERS}; do - # Check to see if operation should be skipped - if (perform_operation "$FOLDER"); then - echo "Opted out of DevOps folder: $FOLDER" - continue + + if ! (check_prs); then + # No existing open pull requests found for this repository + + # Remove remote branch if it exists + git push origin --delete "$AUTOMATION_BRANCH" > /dev/null 2>&1 || : + git branch -D "$AUTOMATION_BRANCH" || : + git checkout -b "$AUTOMATION_BRANCH" + else + # The -B flag swaps branch and creates it if NOT present + git checkout -B "$AUTOMATION_BRANCH" + fi + + # Only if NOT running in GitHub + # (checkout is otherwise performed by earlier steps) + if ! (in_github); then + # Remove any stale local copy of the upstream repository + if [ -d "$DEVOPS_DIR" ]; then + rm -Rf "$DEVOPS_DIR" + fi + printf "Cloning DevOps repository into: %s" "$DEVOPS_DIR" + if (git clone "$DEVOPS_REPO" "$DEVOPS_DIR" > /dev/null 2>&1); then + echo " [success]" else - # If necessary, create target folder - if [ ! -d "$FOLDER" ]; then - echo "Creating target folder: $FOLDER" - mkdir "$FOLDER" - fi - # Update folder contents - echo "Updating folder contents: $FOLDER" - cp -a .devops/"$FOLDER"/. "$FOLDER" + echo " [failed]"; exit 1 fi - done + fi + + # Process upstream DevOps repository content and update + + LOCATIONS="" + # Populate list of files to be updated/sourced + while read -ra LINE; + do + for FILE in "${LINE[@]}"; + do + LOCATIONS+="$FILE " + done + done < "$FILES" + + # Gather files from specified folders and append to locations list + while read -ra LINE; + do + for FOLDER in "${LINE[@]}"; + do + # Check to see if this folder should be skipped + if (perform_folder_operation "$FOLDER"); then + # If necessary, create target folder + if [ ! -d "$FOLDER" ]; then + echo "Creating target folder: $FOLDER" + mkdir "$FOLDER" + fi + # Add folder contents to list of file LOCATIONS + FILES=$(cd "$DEVOPS_DIR/$FOLDER"; find . -maxdepth 1 -type f -exec basename {} \;) + for LOCATION in $FILES; do + # Also check if individual files in the folder are excluded + if (perform_operation "$FOLDER/$LOCATION"); then + LOCATIONS+=" $FOLDER/$LOCATION" + fi + done + else + echo "Opted out of folder: $FOLDER" + continue + fi + done; + done < "$FOLDERS" # Copy specified files into repository root - FILES=".pre-commit-config.yaml .prettierignore .gitignore" - for FILE in ${FILES}; do - if (perform_operation "$FILE"); then - echo "Opted out of DevOps file: $FILE" + for LOCATION in ${LOCATIONS}; do + if (perform_operation "$LOCATION"); then + selective_file_copy "$LOCATION" else - echo "Copying file: $FILE" - cp .devops/"$FILE" "$FILE" + echo "Not updating: $LOCATION" fi done # If no changes required, do not throw an error if [ -z "$(git status --porcelain)" ]; then echo "No updates/changes to commit"; exit 0 - else - # Set a flag for use by the next action/step + fi + + # Temporarily disable exit on unbound variable + set +eu +o pipefail + + # Next step is only performed if running as GitHub Action + if [[ -n ${GH_TOKEN+x} ]]; then + # Script is running in a GitHub actions workflow + # Set outputs for use by the next actions/steps + # shellcheck disable=SC2129 echo "changed=true" >> "$GITHUB_OUTPUT" + echo "branchname=$AUTOMATION_BRANCH" >> "$GITHUB_OUTPUT" + echo "headbranch=$HEAD_BRANCH" >> "$GITHUB_OUTPUT" + # Move to the next workflow step to raise the PR + git push --set-upstream origin "$AUTOMATION_BRANCH" + exit 0 fi - if [ -n "$GITHUB_TOKEN" ]; then - git add . - if ! (git commit -as -S -m "Chore: Update DevOps tooling from central repository [skip-ci]" \ - -m "This commit created by automation/scripting" --no-verify); then - echo "Commit failed; aborting"; exit 1 - else - git push --set-upstream origin update-devops-tooling - # ToDo: need to verify if we are running in a GHA - gh pr create --title \ - "Chore: Pull DevOps tooling from upstream repository" \ - --body 'Automated by a GitHub workflow: bootstrap.yaml' - fi + + # If running shell code locally, continue to raise the PR + + # Reinstate exit on unbound variables + set -euo pipefail + + git status + if ! (git commit -as -S -m "Chore: Update DevOps tooling from central repository [skip ci]" \ + -m "This commit created by automation/scripting" --no-verify); then + echo "Commit failed; aborting"; exit 1 else - echo "Script running in GitHub Actions workflow; proceeding to next step" + # Push branch to remote repository + git push --set-upstream origin "$AUTOMATION_BRANCH" + # Create PR request + gh pr create \ + --title "Chore: Pull DevOps tooling from upstream repository" \ + --body 'Automated by a GitHub workflow: bootstrap.yaml' fi - ### SHELL CODE END ### + # echo "Unstashing unstaged changes, if any exist" + # git stash pop -q || : + #SHELLCODEEND - name: Create Pull Request if: steps.update-repository.outputs.changed == 'true' - uses: peter-evans/create-pull-request@v5 - env: - GITHUB_TOKEN: ${{ github.token }} + uses: peter-evans/create-pull-request@v6 + # env: + # GITHUB_TOKEN: ${{ github.token }} with: - token: ${{ github.token }} - commit-message: "Chore: Update DevOps tooling from central repository [skip-ci]" + # Note: Requires a specific/defined Personal Access Token + token: ${{ secrets.ACTIONS_WORKFLOW }} + commit-message: "Chore: Update DevOps tooling from central repository [skip ci]" signoff: "true" - branch: update-devops-tooling + base: ${{ steps.update-repository.outputs.headbranch }} + branch: ${{ steps.update-repository.outputs.branchname }} delete-branch: true - title: "Chore: Update DevOps tooling from central repository [skip-ci]" + title: "Chore: Update DevOps tooling from central repository [skip ci]" body: | Update repository with content from upstream: os-climate/devops-toolkit labels: | diff --git a/.github/workflows/builds.yaml b/.github/workflows/builds.yaml index 8de5ef0..1c7f171 100644 --- a/.github/workflows/builds.yaml +++ b/.github/workflows/builds.yaml @@ -1,5 +1,5 @@ --- -name: "🧪 Test builds (Matrix)" +name: "🧱 Builds (Matrix)" # yamllint disable-line rule:truthy on: @@ -11,15 +11,21 @@ on: - "!update-devops-tooling" jobs: - pre-release: + parse-project-metadata: + name: "Determine Python versions" + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + + test-builds: + name: "Build: Python" + needs: [parse-project-metadata] runs-on: "ubuntu-latest" continue-on-error: true # Don't run when pull request is merged if: github.event.pull_request.merged == false strategy: fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} steps: - name: "Populate environment variables" @@ -41,10 +47,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: "Install dependencies" - run: | - python -m pip install --upgrade pip - pip install tox tox-gh-actions + - name: "Setup PDM for build commands" + uses: pdm-project/setup-pdm@v4 + with: + python-version: ${{ matrix.python-version }} - name: "Tag for test release" # Delete all local tags, then create a synthetic tag for testing @@ -56,6 +62,27 @@ jobs: git checkout "tags/v${{ steps.setenv.outputs.vernum }}" grep version pyproject.toml - - name: "Build with TOX" + - name: "Performing build" + run: | + python -m pip install --upgrade pip + if [ -f tox.ini ]; then + pip install tox tox-gh-actions + echo "Found file: tox.ini" + echo "Building with command: tox -e build" + tox -e build + elif [ -f pyproject.toml ]; then + echo "Found file: pyproject.toml" + echo "Building with command: pdm build" + pdm build + else + echo "Neither file found: tox.ini/pyproject.toml" + pip install --upgrade build + echo "Attempting build with: python -m build" + python -m build + fi + + - name: "Validating Artefacts with Twine" run: | - tox -e build + echo "Validating artefacts with: twine check dist/*" + pip install --upgrade twine + twine check dist/* diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 3796a60..09a0ffa 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -1,5 +1,5 @@ --- -name: "🗒️ Build documentation (Matrix)" +name: "📘 Documentation build/publish" # yamllint disable-line rule:truthy on: @@ -12,31 +12,30 @@ on: jobs: build_and_deploy: - # Don't run if pull request is NOT merged + # Only run when pull request is merged if: github.event.pull_request.merged == true name: "Rebuild documentation" runs-on: ubuntu-latest continue-on-error: true - strategy: - matrix: - python-version: ["3.11"] + permissions: # IMPORTANT: mandatory for documentation updates; used in final step id-token: write pull-requests: write contents: write repository-projects: write + steps: - name: "Checkout repository" uses: actions/checkout@v4 - - name: "Set up Python ${{ matrix.python-version }}" + - name: "Set up Python" uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 - name: "Install dependencies" run: | @@ -48,11 +47,12 @@ jobs: - name: "Build documentation: (tox/sphinx)" run: | + pip install --upgrade tox tox -e docs - name: "Publish documentation" if: success() - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: publish_branch: gh-pages github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/notebooks.yaml b/.github/workflows/notebooks.yaml new file mode 100644 index 0000000..0fad0cc --- /dev/null +++ b/.github/workflows/notebooks.yaml @@ -0,0 +1,84 @@ +--- +# Run all notebooks on every push +name: "🗒️ Notebook Testing" + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + + validate-notebook-tests: + name: "Check for notebook tests" + if: github.event.pull_request.merged == false + permissions: + actions: 'write' + runs-on: ubuntu-latest + outputs: + proceed: ${{ steps.proceed.outputs.proceed }} + steps: + - name: "Checking for test files" + id: proceed + run: | + RESULT=(**/test_*.ipynb) + if [ -z "${RESULT[0]}" ]; then + echo "Found Jupyter Notebook tests" + echo "proceed=true" >> "$GITHUB_OUTPUT" + fi + + parse-project-metadata: + name: "Determine Python versions" + needs: [validate-notebook-tests] + if: needs.validate-notebook-tests.outputs.proceed == 'true' + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + + notebook-tests: + name: "Test Jupyter Notebooks" + needs: [validate-notebook-tests, parse-project-metadata] + runs-on: ubuntu-latest + continue-on-error: true + # Don't run when pull request is merged + if: github.event.pull_request.merged == 'false' + + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Set up Python ${{ matrix.python-version }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: "Setup PDM for build commands" + uses: pdm-project/setup-pdm@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: "Install dependencies" + run: | + which python; which python3 + python --version; python3 --version + python -m pip install --upgrade pip + pdm export -o requirements.txt + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install . + pip install pytest nbmake + + - name: "Testing Jupyter Notebooks" + run: | + echo "Testing notebooks using: pytest --nbmake -- **/test_*.ipynb" + pytest --nbmake -- **/test_*.ipynb + + - name: Upload logs as artefacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: debug-logs + path: /tmp/*.log + retention-days: 14 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7b99e21..ac254c4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,8 +9,11 @@ on: # workflow_dispatch: push: # Only invoked on release tag pushes + branches: + - 'main' + - 'master' tags: - - v*.*.* + - 'v*.*.*' env: python-version: "3.10" @@ -18,8 +21,11 @@ env: ### BUILD ### jobs: + build: name: "🐍 Build packages" + # Only publish on tag pushes + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest permissions: # IMPORTANT: mandatory for Sigstore @@ -36,7 +42,7 @@ jobs: python-version: ${{ env.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 - name: "Update version from tags for production release" run: | @@ -50,7 +56,7 @@ jobs: ### SIGNING ### - name: "Sign packages with Sigstore" - uses: sigstore/gh-action-sigstore-python@v2.1.0 + uses: sigstore/gh-action-sigstore-python@v2 with: inputs: >- ./dist/*.tar.gz @@ -81,15 +87,16 @@ jobs: name: ${{ github.ref_name }} path: dist/ - - name: "📦 Publish release to GitHub" - uses: ModeSevenIndustrialSolutions/action-automatic-releases@latest + - name: "📦 Publish artefacts to GitHub" + # https://github.com/softprops/action-gh-release + uses: softprops/action-gh-release@v2 with: - # Valid inputs are: - # repo_token, automatic_release_tag, draft, prerelease, title, files - repo_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} prerelease: false - automatic_release_tag: ${{ github.ref_name }} - title: ${{ github.ref_name }} + tag_name: ${{ github.ref_name }} + name: "Test/Development Build \ + ${{ github.ref_name }}" + # body_path: ${{ github.workspace }}/CHANGELOG.rst files: | dist/*.tar.gz dist/*.whl @@ -158,7 +165,7 @@ jobs: rm dist/*.sigstore - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 - name: "Publish release to PyPI" uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/security.yaml b/.github/workflows/security.yaml index 9d34824..26251da 100644 --- a/.github/workflows/security.yaml +++ b/.github/workflows/security.yaml @@ -16,15 +16,21 @@ on: - "!update-devops-tooling" jobs: + + parse-project-metadata: + name: "Determine Python versions" + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + build: name: "Audit Python dependencies" + needs: [parse-project-metadata] runs-on: ubuntu-latest # Don't run when pull request is merged if: github.event.pull_request.merged == false strategy: fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} steps: - name: "Checkout repository" @@ -36,7 +42,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 with: python-version: ${{ matrix.python-version }} @@ -47,6 +53,7 @@ jobs: pdm export -o requirements.txt python -m pip install -r requirements.txt python -m pip install . + pip install --upgrade setuptools pdm list --graph - name: "Run: pip-audit" diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index efcd64d..bafecb4 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -6,7 +6,7 @@ name: "🐍📦 Test build and release" # yamllint disable-line rule:truthy on: - workflow_dispatch + workflow_dispatch: env: python-version: "3.10" @@ -32,7 +32,7 @@ jobs: python-version: ${{ env.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 with: python-version: ${{ env.python-version }} @@ -62,7 +62,8 @@ jobs: ### SIGNING ### - name: "Sign packages with Sigstore" - uses: sigstore/gh-action-sigstore-python@v2.1.0 + uses: sigstore/gh-action-sigstore-python@v2 + with: inputs: >- ./dist/*.tar.gz @@ -103,16 +104,16 @@ jobs: echo "tarball=$(ls dist/*.tgz)" >> "$GITHUB_OUTPUT" echo "wheel=$(ls dist/*.whl)" >> "$GITHUB_OUTPUT" - - name: "📦 Publish packages to GitHub" - uses: ModeSevenIndustrialSolutions/action-automatic-releases@latest + - name: "📦 Publish artefacts to GitHub" + # https://github.com/softprops/action-gh-release + uses: softprops/action-gh-release@v2 with: - # Valid inputs are: - # repo_token, automatic_release_tag, draft, prerelease, title, files - repo_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} prerelease: true - automatic_release_tag: ${{ steps.setenv.outputs.vernum }} - title: "Development Build \ + tag_name: ${{ steps.setenv.outputs.vernum }} + name: "Test/Development Build \ ${{ steps.setenv.outputs.vernum }}" + # body_path: ${{ github.workspace }}/CHANGELOG.rst files: | dist/*.tar.gz dist/*.whl diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index bb80cec..aebd8a9 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -11,15 +11,21 @@ on: - "!update-devops-tooling" jobs: - build: + + parse-project-metadata: + name: "Determine Python versions" + # yamllint disable-line rule:line-length + uses: os-climate/devops-reusable-workflows/.github/workflows/pyproject-toml-fetch-matrix.yaml@main + + testing: name: "Run unit tests" + needs: [parse-project-metadata] runs-on: ubuntu-latest # Don't run when pull request is merged if: github.event.pull_request.merged == false strategy: fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] + matrix: ${{ fromJson(needs.parse-project-metadata.outputs.matrix) }} steps: - name: "Checkout repository" @@ -31,7 +37,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: "Setup PDM for build commands" - uses: pdm-project/setup-pdm@v3 + uses: pdm-project/setup-pdm@v4 with: python-version: ${{ matrix.python-version }} @@ -40,6 +46,7 @@ jobs: python -m pip install --upgrade pip pdm export -o requirements.txt pip install -r requirements.txt + pip install --upgrade pytest pytest-cov pip install . - name: "Run unit tests: pytest" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d5293d..9811726 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,6 @@ ci: skip: # pre-commit.ci cannot install WGET, so tomlint must be disabled - tomllint - # - pre-commit-update exclude: | (?x)^( @@ -14,12 +13,6 @@ exclude: | repos: - # - repo: https://gitlab.com/vojko.pribudic/pre-commit-update - # rev: v0.1.0 - # hooks: - # - id: pre-commit-update - # args: [--dry-run] - - repo: local hooks: - id: tomllint @@ -31,7 +24,7 @@ repos: entry: scripts/tomllint.sh . - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-ast @@ -52,42 +45,36 @@ repos: args: ["--fix=lf"] - id: name-tests-test args: ["--pytest-test-first"] - # Do not allow direct push to main/master branches - id: no-commit-to-branch # - id: pretty-format-json - id: requirements-txt-fixer - id: trailing-whitespace # Autoformat: YAML, JSON, Markdown, etc. - - repo: https://github.com/prettier/pre-commit - rev: v2.2.0 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 hooks: - id: prettier args: - ['--ignore-unknown'] + ['--no-error-on-unmatched-pattern', '--ignore-unknown'] # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.38.0 + rev: v0.41.0 hooks: - id: markdownlint args: ["--fix"] - # - repo: https://github.com/asottile/pyupgrade - # rev: v3.15.0 + # - repo: https://github.com/psf/black-pre-commit-mirror + # rev: 24.4.2 # hooks: - # - id: pyupgrade - # args: ['--py37-plus'] + # - id: black + # - id: black-jupyter - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.1 - hooks: - - id: black - - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.1 - hooks: - - id: black-jupyter + # - repo: https://github.com/tomcatling/black-nb + # rev: '0.7' + # hooks: + # - id: black-nb - repo: https://github.com/jorisroovers/gitlint rev: v0.19.1 @@ -101,22 +88,15 @@ repos: args: ["--ignore=E006,E011"] - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck - # Optionally only show errors and warnings - # args: ["--severity=warning"] - # If you want to avoid flake8 errors due to unused vars or imports: - # - repo: https://github.com/PyCQA/autoflake - # rev: v2.0.0 - # hooks: - # - id: autoflake - # args: [ - # --in-place, - # --remove-all-unused-imports, - # --remove-unused-variables, - # ] + - repo: https://github.com/pycqa/pydocstyle.git + rev: 6.3.0 + hooks: + - id: pydocstyle + additional_dependencies: ["tomli"] - repo: https://github.com/PyCQA/isort rev: 5.13.2 @@ -124,41 +104,57 @@ repos: - id: isort args: ["--profile", "black"] - - repo: https://github.com/adrienverge/yamllint.git - rev: v1.33.0 - hooks: - - id: yamllint - args: [--strict] - - repo: https://github.com/Mateusz-Grzelinski/actionlint-py - rev: v1.6.26.11 + rev: v1.7.0.14 hooks: - id: actionlint - # If like to embrace black styles even in the docs: - # - repo: https://github.com/asottile/blacken-docs - # rev: v1.13.0 - # hooks: - # - id: blacken-docs - # additional_dependencies: [black] - - repo: https://github.com/pycqa/flake8 - rev: "6.1.0" + rev: "7.0.0" hooks: - id: flake8 - entry: pflake8 - additional_dependencies: [pyproject-flake8] + additional_dependencies: + - pep8-naming - # Check for misspells in documentation files: - # - repo: https://github.com/codespell-project/codespell - # rev: v2.2.2 - # hooks: - # - id: codespell + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint + args: [ "-d", "{rules: {line-length: {max: 120}}, ignore-from-file: [.gitignore],}", ] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.5 + hooks: + - id: ruff + files: ^(scripts|tests|custom_components)/.+\.py$ + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + files: ^(scripts|tests|custom_components)/.+\.py$ - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.8.0" + rev: "v1.10.0" hooks: - id: mypy verbose: true - args: [--show-error-codes] - additional_dependencies: ["pytest", "types-requests", "types-PyYAML"] + args: ["--show-error-codes", "--install-types", "--non-interactive"] + additional_dependencies: ["pytest", "types-requests"] + + # Check for misspellings in documentation files + # - repo: https://github.com/codespell-project/codespell + # rev: v2.2.2 + # hooks: + # - id: codespell + + # To embrace black styles, even in docs + # - repo: https://github.com/asottile/blacken-docs + # rev: v1.13.0 + # hooks: + # - id: blacken-docs + # additional_dependencies: [black] + + # Automatically upgrade Python syntax for newer versions + # - repo: https://github.com/asottile/pyupgrade + # rev: v3.15.0 + # hooks: + # - id: pyupgrade + # args: ['--py37-plus']