diff --git a/.github/has-functional-changes.sh b/.github/has-functional-changes.sh new file mode 100644 index 00000000..01a4b219 --- /dev/null +++ b/.github/has-functional-changes.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*" + +last_tagged_commit=`git describe --tags --abbrev=0 --first-parent` # --first-parent ensures we don't follow tags not published in main through an unlikely intermediary merge commit + +if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'` # Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published. +then + echo "No functional changes detected." + exit 1 +else echo "The functional files above were changed." +fi \ No newline at end of file diff --git a/.github/is-version-number-acceptable.sh b/.github/is-version-number-acceptable.sh new file mode 100644 index 00000000..d68219e2 --- /dev/null +++ b/.github/is-version-number-acceptable.sh @@ -0,0 +1,39 @@ +#! /usr/bin/env bash + +if [[ ${GITHUB_REF#refs/heads/} == main ]] +then + echo "No need for a version check on main." + exit 0 +fi + +if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh +then + echo "No need for a version update." + exit 0 +fi + +current_version=$(grep '^version =' pyproject.toml | cut -d '"' -f 2) # parsing with tomllib is complicated, see https://github.com/python-poetry/poetry/issues/273 + +if [[ ! $current_version ]] +then + echo "Error getting current version" + exit 1 +fi + +if git rev-parse --verify --quiet $current_version +then + echo "Version $current_version already exists in commit:" + git --no-pager log -1 $current_version + echo + echo "Update the version number in pyproject.toml before merging this branch into main." + echo "Look at the CONTRIBUTING.md file to learn how the version number should be updated." + exit 2 +fi + +if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh | grep --quiet CHANGELOG.md +then + echo "CHANGELOG.md has not been modified, while functional changes were made." + echo "Explain what you changed before merging this branch into main." + echo "Look at the CONTRIBUTING.md file to learn how to write the changelog." + exit 2 +fi \ No newline at end of file diff --git a/.github/lint-files.sh b/.github/lint-files.sh new file mode 100644 index 00000000..13c92980 --- /dev/null +++ b/.github/lint-files.sh @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +# Usage: lint-files.sh +# +# Example usage: +# lint-files.sh "*.py" "flake8" +# lint-files.sh "tests/*.yaml" "yamllint" + +file_pattern=$1 +linter_command=$2 + +if [ -z "$file_pattern" ] || [ -z "$linter_command" ] +then + echo "Usage: $0 " + exit 1 +fi + +last_tagged_commit=$(git describe --tags --abbrev=0 --first-parent 2>/dev/null) # Attempt to find the last tagged commit in the direct ancestry of the main branch avoiding tags introduced by merge commits from other branches + +if [ -z "$last_tagged_commit" ] +then + last_tagged_commit=$(git rev-list --max-parents=0 HEAD) # Fallback to finding the root commit if no tags are present +fi + +if ! changed_files=$(git diff-index --name-only --diff-filter=ACMR --exit-code $last_tagged_commit -- "$file_pattern") +then + echo "Linting the following files:" + echo "$changed_files" + $linter_command $changed_files +else + echo "No changed files matching pattern '$file_pattern' to lint." +fi \ No newline at end of file diff --git a/.github/publish-git-tag.sh b/.github/publish-git-tag.sh new file mode 100644 index 00000000..c02eee86 --- /dev/null +++ b/.github/publish-git-tag.sh @@ -0,0 +1,5 @@ +#! /usr/bin/env bash + +current_version=$(grep '^version =' pyproject.toml | cut -d '"' -f 2) # parsing with tomllib is complicated, see https://github.com/python-poetry/poetry/issues/273 +git tag $current_version +git push --tags # update the repository version \ No newline at end of file diff --git a/.github/test-api.sh b/.github/test-api.sh new file mode 100644 index 00000000..54c93859 --- /dev/null +++ b/.github/test-api.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env bash + +PORT=5000 +ENDPOINT=spec + +openfisca serve --country-package openfisca_country_template --port $PORT & +server_pid=$! + +curl --retry-connrefused --retry 10 --retry-delay 5 --fail http://127.0.0.1:$PORT/$ENDPOINT | python -m json.tool > /dev/null +result=$? + +kill $server_pid + +exit $? \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..7e7f5f3f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,34 @@ +name: Build + +on: + workflow_call: + +jobs: + build-and-cache: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 # Patch version must be specified to avoid any cache confusion, since the cache key depends on the full Python version. Any difference in patches between jobs will lead to a cache not found error. + + - name: Cache build + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} # Cache the entire build Python environment + restore-keys: | + build-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }} + build-${{ env.pythonLocation }}- + + - name: Build package + run: make build + + - name: Cache release + uses: actions/cache@v4 + with: + path: dist + key: release-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..eb4f605a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,70 @@ +ame: Deploy + +on: + push: + branches: [ main ] + +jobs: + validate: + uses: "./.github/workflows/validate.yml" + + # GitHub Actions does not have a halt job option to stop from deploying if no functional changes were found. + # We thus execute a separate deployment job depending on the output of this job. + check-for-functional-changes: + runs-on: ubuntu-22.04 + outputs: + status: ${{ steps.stop-early.outputs.status }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all the tags + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 + - id: stop-early + run: | + if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" + then + echo "status=success" >> $GITHUB_OUTPUT + fi + + check-pypi-token: # Use intermediary job as secrets cannot be directly referenced in `if:` conditionals; see https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#using-secrets-in-a-workflow + runs-on: ubuntu-22.04 + outputs: + pypi_token_present: ${{ steps.check_token.outputs.pypi_token_present }} + steps: + - name: Check PyPI token is defined + id: check_token + run: | + if [[ -n "${{ secrets.PYPI_TOKEN }}" ]] + then + echo "pypi_token_present=true" >> $GITHUB_OUTPUT + else + echo "pypi_token_present=false" >> $GITHUB_OUTPUT + fi + + deploy: + runs-on: ubuntu-22.04 + needs: [ validate, check-for-functional-changes, check-pypi-token ] + if: needs.check-for-functional-changes.outputs.status == 'success' && needs.check-pypi-token.outputs.pypi_token_present == 'true' + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 + - name: Restore build + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} + - name: Restore built package + uses: actions/cache@v4 + with: + path: dist + key: release-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} + - name: Upload a Python package to PyPi + run: twine upload dist/* --username __token__ --password ${{ secrets.PYPI_TOKEN }} + - name: Publish a git tag + run: "${GITHUB_WORKSPACE}/.github/publish-git-tag.sh" \ No newline at end of file diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml deleted file mode 100644 index 2cd76f35..00000000 --- a/.github/workflows/python.yaml +++ /dev/null @@ -1,67 +0,0 @@ -name: Python - -on: - push: - branches: [main] - pull_request: - types: [assigned, opened, reopened, synchronize, ready_for_review] - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.11.9 - - - name: Cache tests - uses: actions/cache@v3 - with: - path: | - Makefile - openfisca_aotearoa/tests - key: tests-${{ github.sha }} - - - name: Cache build - id: cache-build - uses: actions/cache@v3 - with: - path: dist - key: build-${{ github.sha }} - - - name: Build package - if: steps.cache-build.outputs.cache-hit != 'true' - run: make install - - test: - runs-on: ubuntu-latest - needs: [build] - - steps: - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.11.9 - - - name: Restore tests - uses: actions/cache@v3 - with: - path: | - Makefile - openfisca_aotearoa/tests - key: tests-${{ github.sha }} - - - name: Restore build - uses: actions/cache@v3 - with: - path: dist - key: build-${{ github.sha }} - - - name: Run the test suite - run: make test diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..bc8acdb8 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,93 @@ +name: Validate + +on: + pull_request: + types: [ assigned, opened, reopened, synchronize, ready_for_review ] + workflow_call: + +jobs: + build: + uses: "./.github/workflows/build.yml" + + lint-files: + runs-on: ubuntu-22.04 + needs: [ build ] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 + + - name: Restore build + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} + + - run: make check-syntax-errors + + - run: make check-style + + - name: Lint Python files + run: "${GITHUB_WORKSPACE}/.github/lint-files.sh '*.py' 'flake8'" + + - name: Lint YAML tests + run: "${GITHUB_WORKSPACE}/.github/lint-files.sh 'tests/*.yaml' 'yamllint'" + + test-yaml: + runs-on: ubuntu-22.04 + needs: [ build ] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 + + - name: Restore build + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} + + - run: make test + + test-api: + runs-on: ubuntu-22.04 + needs: [ build ] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 + + - name: Restore build + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: build-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ github.sha }} + + - name: Test the Web API + run: "${GITHUB_WORKSPACE}/.github/test-api.sh" + + check-version-and-changelog: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all the tags + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.9 + + - name: Check version number has been properly updated + run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" \ No newline at end of file diff --git a/Makefile b/Makefile index c4e20857..583a0a10 100644 --- a/Makefile +++ b/Makefile @@ -26,21 +26,21 @@ build: clean deps check-syntax-errors: python -m compileall -q . -format: +format-style: @# Do not analyse .gitignored files. @# `make` needs `$$` to output `$`. Ref: http://stackoverflow.com/questions/2382764. isort `git ls-files | grep "\.py$$"` autopep8 `git ls-files | grep "\.py$$"` pyupgrade --py39-plus `git ls-files | grep "\.py$$"` -lint: clean check-syntax-errors format +check-style: @# Do not analyse .gitignored files. @# `make` needs `$$` to output `$`. Ref: http://stackoverflow.com/questions/2382764. flake8 `git ls-files | grep "\.py$$"` pylint `git ls-files | grep "\.py$$"` yamllint `git ls-files | grep "\.yaml$$"` -test: lint +test: clean check-syntax-errors check-style ifdef yaml openfisca test -c openfisca_aotearoa openfisca_aotearoa/tests/$(yaml) else