Add code coverage

.github/workflows/test.yml

@@ -10,23 +10,25 @@ concurrency:
 
 jobs:
   tests:
-    name: Python ${{ matrix.python-version }}, Database ${{ matrix.database-type }}
+    name: Python ${{ matrix.python-version[0] }}, Database ${{ matrix.database-type }}
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
         python-version:
-          - "3.7"
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          - "pypy-3.7"
-          - "pypy-3.8"
-          - "pypy-3.9"
+          - ["3.7", "py37"]
+          - ["3.8", "py38"]
+          - ["3.9", "py39"]
+          - ["3.10", "py310"]
+          - ["pypy-3.7", "pypy37"]
+          - ["pypy-3.8", "pypy38"]
+          - ["pypy-3.9", "pypy39"]
         database-type:
           - "sqlite"
           - "postgres"
+        run-coverage:
+          - "${{ (github.ref == 'refs/heads/master' || contains(github.event.commits[0].message, 'coverage')) && 'yes' || 'no' }}"
 
     services:
       mongodb:
@@ -41,16 +43,13 @@ jobs:
         env:
           POSTGRES_PASSWORD: password
 
-    env:
-      TOXENV: ${{ matrix.tox-environment }}
-
     steps:
       - uses: actions/checkout@v2
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python ${{ matrix.python-version[0] }}
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python-version[0] }}
 
       - name: Install dependencies
         run: python -m pip install tox-gh-actions
@@ -59,3 +58,16 @@ jobs:
         run: tox
         env:
           DATABASE_TYPE: ${{ matrix.database-type }}
+          RUN_COVERAGE: ${{ matrix.run-coverage }}
+
+      - name: Coverage report
+        if: matrix.run-coverage == 'yes'
+        run: tox -e cov-report
+
+      - name: 'Upload coverage'
+        if: matrix.run-coverage == 'yes'
+        uses: actions/upload-artifact@v3
+        with:
+          name: factory_coverage_${{ matrix.python-version[1] }}_${{ matrix.database-type }}
+          path: htmlcov/
+          retention-days: 1

Makefile

@@ -3,9 +3,21 @@ TESTS_DIR=tests
 DOC_DIR=docs
 EXAMPLES_DIR=examples
 SETUP_PY=setup.py
+PYTHON_ERROR_ON_WARN=python \
+	-b \
+	-X dev \
+	-Werror \
+	-Wdefault:"the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses":DeprecationWarning:distutils: \
+	-Wdefault:"Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working":DeprecationWarning:: \
+	-Wdefault:"Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.9 it will stop working":DeprecationWarning:: \
+	-Wdefault:"set_output_charset() is deprecated":DeprecationWarning:: \
+	-Wdefault:"parameter codeset is deprecated":DeprecationWarning:: \
+	-Wdefault:"distutils Version classes are deprecated. Use packaging.version instead":DeprecationWarning::
+# TODO: Remove "distutils Version classes are deprecated" when django 2.2 is dropped
 
 # Use current python binary instead of system default.
-COVERAGE = python $(shell which coverage)
+COVERAGE_PATH = $(shell which coverage)
+COVERAGE = python $(COVERAGE_PATH)
 FLAKE8 = flake8
 ISORT = isort
 CTAGS = ctags
@@ -53,18 +65,7 @@ testall:
 
 # DOC: Run tests for the currently installed version
 test:
-	python \
-		-b \
-		-X dev \
-		-Werror \
-		-Wdefault:"the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses":DeprecationWarning:distutils: \
-		-Wdefault:"Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working":DeprecationWarning:: \
-		-Wdefault:"Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3, and in 3.9 it will stop working":DeprecationWarning:: \
-		-Wdefault:"set_output_charset() is deprecated":DeprecationWarning:: \
-		-Wdefault:"parameter codeset is deprecated":DeprecationWarning:: \
-		-Wdefault:"distutils Version classes are deprecated. Use packaging.version instead":DeprecationWarning:: \
-		-m unittest
-# TODO: Remove "distutils Version classes are deprecated" when django 2.2 is dropped
+	$(PYTHON_ERROR_ON_WARN) -m unittest
 
 # DOC: Test the examples
 example-test:
@@ -80,14 +81,19 @@ lint:
 	$(ISORT) --check-only --diff $(EXAMPLES_DIR) $(PACKAGE) $(SETUP_PY) $(TESTS_DIR)
 	check-manifest
 
-coverage:
+coverage-clean:
 	$(COVERAGE) erase
-	$(COVERAGE) run "--include=$(PACKAGE)/*.py,$(TESTS_DIR)/*.py" --branch -m unittest
-	$(COVERAGE) report "--include=$(PACKAGE)/*.py,$(TESTS_DIR)/*.py"
-	$(COVERAGE) html "--include=$(PACKAGE)/*.py,$(TESTS_DIR)/*.py"
+	@rm -rf .coverage htmlcov/
+
+coverage-test:
+	$(PYTHON_ERROR_ON_WARN) $(COVERAGE_PATH) run -m unittest
+
+coverage-report:
+	$(COVERAGE) report
+	$(COVERAGE) html
 
 
-.PHONY: test testall example-test lint coverage
+.PHONY: test testall example-test lint coverage-clean coverage-test coverage-report
 
 
 # Development

setup.cfg

@@ -81,3 +81,17 @@ include_trailing_comma = True
 force_grid_wrap = 0
 use_parentheses = True
 line_length = 88
+
+[coverage:run]
+dynamic_context = test_function
+include=
+    factory/*.py
+    tests/*.py
+
+[coverage:report]
+include=
+    factory/*.py
+    tests/*.py
+
+[coverage:html]
+show_contexts = True

tox.ini

@@ -5,11 +5,11 @@ envlist =
     docs
     examples
     linkcheck
-    py{37,38,39,310,py37,py38}-sqlite
-    py{37,38,39,py37,py38}-django22-mongo-alchemy-{sqlite,postgres}
-    py{37,38,39,310,py37,py38}-django32-mongo-alchemy-{sqlite,postgres}
-    py{38,39,310,py38}-django40-mongo-alchemy-{sqlite,postgres}
-    py310-djangomain-mongo-alchemy-{sqlite,postgres}
+    py{37,38,39,310,py37,py38,py39}-sqlite-{cov,nocov}
+    py{37,38,39,py37,py38,py39}-django22-mongo-alchemy-{sqlite,postgres}-{cov,nocov}
+    py{37,38,39,310,py37,py38,py39}-django32-mongo-alchemy-{sqlite,postgres}-{cov,nocov}
+    py{38,39,310,py38,py39}-django40-mongo-alchemy-{sqlite,postgres}-{cov,nocov}
+    py310-djangomain-mongo-alchemy-{sqlite,postgres}-{cov,nocov}
 
 [gh-actions]
 python =
@@ -25,13 +25,17 @@ python =
 DATABASE_TYPE =
     sqlite: sqlite
     postgres: postgres
+RUN_COVERAGE =
+    yes: cov
+    no: nocov
 
 [testenv]
 passenv =
     MONGO_HOST
     POSTGRES_HOST
     POSTGRES_DATABASE
 deps =
+    cov: coverage
     alchemy: SQLAlchemy
     alchemy: sqlalchemy_utils
     mongo: mongoengine
@@ -41,14 +45,24 @@ deps =
     django40: Django>=4.0,<4.1
     djangomain: https://github.com/django/django/archive/main.tar.gz
     py{37,38,39,310}-postgres: psycopg2-binary
-    py{py37,py38}-postgres: psycopg2cffi
+    pypy{37,38,39}-postgres: psycopg2cffi
 
 setenv =
-    py: DJANGO_SETTINGS_MODULE=tests.djapp.settings
+    py-!postgres: DJANGO_SETTINGS_MODULE=tests.djapp.settings
     postgres: DJANGO_SETTINGS_MODULE=tests.djapp.settings_pg
 
 whitelist_externals = make
-commands = make test
+commands =
+    !cov: make test
+    cov: make coverage-test
+
+[testenv:cov-clean]
+commands =
+    make coverage-clean
+
+[testenv:cov-report]
+commands =
+    make coverage-report
 
 [testenv:docs]
 extras = doc