Enable warnings as errors for weekly run in test suite

.github/workflows/interface-unit-tests.yml

@@ -76,6 +76,14 @@ on:
         required: false
         type: boolean
         default: false
+      python_warning_level:
+        description: Sets the default Python warning level as defined by https://docs.python.org/3/using/cmdline.html#envvar-PYTHONWARNINGS
+        required: false
+        type: string
+        default: 'default'
+env:
+  ACTIONS_RUNNER_DEBUG: true
+  ACTIONS_STEP_DEBUG: true
 
 jobs:
   determine_runner:
@@ -86,6 +94,34 @@ jobs:
       default_runner: ubuntu-latest
       force_large_runner: ${{ inputs.use_large_runner }}
 
+  disable-fail-fast-wae:
+    needs:
+      - determine_runner
+
+    runs-on: ${{ needs.determine_runner.outputs.runner_group }}
+    steps:
+      - name: Set fail-fast for WAE
+        id: mat_fail_fast
+        run: |
+          if [ "${{ contains(inputs.python_warning_level, 'default') }}" ];
+          then
+            cat >python_warning_level.json <<-EOF
+          {
+            "matrix-fail-fast": "default"
+          }
+          EOF
+          else
+            cat >python_warning_level.json <<-EOF
+          {
+            "matrix-fail-fast": "error"
+          }
+          EOF
+          fi
+          echo "fail_fast=$(jq -r tostring python_warning_level.json)" >> $GITHUB_OUTPUT
+
+    outputs:
+      matrix-fail-fast: ${{ steps.mat_fail_fast.outputs.fail_fast }}
+
   setup-ci-load:
     needs:
       - determine_runner
@@ -106,6 +142,22 @@ jobs:
             "default": ["3.10"]
           }
           EOF
+          elif [ "${{ inputs.python_warning_level }}" == "error" ];
+          then
+            cat >python_versions.json <<-EOF
+          {
+            "default": ["3.10"],
+            "torch-tests": ["3.10"],
+            "tf-tests": ["3.10"],
+            "jax-tests": ["3.10"],
+            "all-interfaces-tests": ["3.10"],
+            "external-libraries-tests": ["3.10"],
+            "qcut-tests": ["3.10"],
+            "qchem-tests": ["3.10"],
+            "gradients-tests": ["3.10"],
+            "data-tests": ["3.10"],
+            "device-tests": ["3.10"]          }
+          EOF
           else
             cat >python_versions.json <<-EOF
           {
@@ -219,7 +271,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).torch-tests
@@ -245,6 +299,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: torch and not qcut and not finite-diff and not param-shift
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'torch.txt' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
 
@@ -254,7 +309,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).autograd-tests
@@ -278,6 +335,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: autograd and not qcut and not finite-diff and not param-shift
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
 
 
@@ -286,7 +344,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).tf-tests
@@ -313,18 +373,19 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: tf and not qcut and not finite-diff and not param-shift
-      pytest_additional_args: --splits 3 --group ${{ matrix.group }}
+      pytest_additional_args: --splits 3 --group ${{ matrix.group }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       pytest_durations_file_path: '.github/durations/tf_tests_durations.json'
       requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'tf.txt' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
 
-
   jax-tests:
     needs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).jax-tests
@@ -351,7 +412,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: jax and not qcut and not finite-diff and not param-shift
-      pytest_additional_args: --dist=loadscope --splits 4 --group ${{ matrix.group }}
+      pytest_additional_args: --dist=loadscope --splits 4 --group ${{ matrix.group }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       pytest_durations_file_path: '.github/durations/jax_tests_durations.json'
       requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'jax.txt' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
@@ -362,7 +423,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).core-tests
@@ -387,7 +450,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: core and not qcut and not finite-diff and not param-shift
-      pytest_additional_args: --splits 6 --group ${{ matrix.group }}
+      pytest_additional_args: --splits 6 --group ${{ matrix.group }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       pytest_durations_file_path: '.github/durations/core_tests_durations.json'
       requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'core.txt' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
@@ -398,7 +461,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).all-interfaces-tests
@@ -426,6 +491,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: all_interfaces
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       requirements_file: ${{ github.event_name == 'schedule' && strategy.job-index == 0 && 'all_interfaces.txt' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
 
@@ -435,7 +501,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).external-libraries-tests
@@ -457,6 +525,7 @@ jobs:
       python_version: ${{ matrix.python-version }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: external
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       additional_pip_packages: |
         pyzx matplotlib stim quimb mitiq pennylane-qiskit ply
         ${{ needs.default-dependency-versions.outputs.jax-version }}
@@ -479,7 +548,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).qcut-tests
@@ -501,6 +572,7 @@ jobs:
       python_version: ${{ matrix.python-version }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: qcut
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       additional_pip_packages: |
         kahypar==1.1.7 
         opt_einsum
@@ -517,7 +589,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).qchem-tests
@@ -541,6 +615,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: qchem
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       additional_pip_packages: |
         openfermionpyscf basis-set-exchange
         ${{ inputs.additional_python_packages }}
@@ -551,7 +626,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).gradients-tests
@@ -582,6 +659,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: ${{ matrix.config.suite }}
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
 
 
@@ -590,7 +668,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).data-tests
@@ -613,6 +693,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
       pytest_markers: data
+      pytest_additional_args: -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       additional_pip_packages: |
         h5py
         ${{ inputs.additional_python_packages }}
@@ -624,7 +705,9 @@ jobs:
       - setup-ci-load
       - determine_runner
       - default-dependency-versions
+      - disable-fail-fast-wae
     strategy:
+      fail-fast: ${{ contains(fromJSON(needs.disable-fail-fast-wae.outputs.matrix-fail-fast),  'default') }}
       max-parallel: >-
         ${{
            fromJSON(needs.setup-ci-load.outputs.matrix-max-parallel).device-tests
@@ -659,7 +742,7 @@ jobs:
       additional_pip_packages_post: ${{ needs.default-dependency-versions.outputs.pennylane-lightning-latest }}
       pytest_test_directory: pennylane/devices/tests
       pytest_coverage_flags: ${{ inputs.pytest_coverage_flags }}
-      pytest_additional_args: --device=${{ matrix.config.device }} --shots=${{ matrix.config.shots }}
+      pytest_additional_args: --device=${{ matrix.config.device }} --shots=${{ matrix.config.shots }} -W ${{ inputs.python_warning_level }} ${{ inputs.python_warning_level != 'default' && '--continue-on-collection-errors' || '' }}
       disable_new_opmath: ${{ inputs.disable_new_opmath }}
 
 

.github/workflows/package_warnings_as_errors.yml

@@ -0,0 +1,31 @@
+name: Test-suite with Python warnings as errors
+on:
+  # Scheduled trigger every Sunday at 2:35am UTC
+  schedule:
+  - cron: '35 2 * * 0'
+  workflow_dispatch:
+
+concurrency:
+  group: warnings-as-errors-tests-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  test-warnings-as-errors:
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+    secrets:
+      codecov_token: ${{ secrets.CODECOV_TOKEN }}
+    uses: ./.github/workflows/interface-unit-tests.yml
+    with:
+      branch: ${{ github.ref }}
+      python_warning_level: 'error'
+      run_lightened_ci: false
+      skip_ci_test_jobs: |
+        tf-tests
+        all-interfaces-tests
+        external-libraries-tests
+        qcut-tests
+        qchem-tests
+        gradients-tests
+        device-tests

doc/releases/changelog-dev.md

@@ -359,6 +359,8 @@
   [(#6244)](https://github.com/PennyLaneAI/pennylane/pull/6244)
 
 <h3>Documentation 📝</h3>
+* Add reporting of test warnings as failures.
+  [(#6217)](https://github.com/PennyLaneAI/pennylane/pull/6217)
 
 * Updated `qml.spin` documentation.
   [(#6387)](https://github.com/PennyLaneAI/pennylane/pull/6387)

pennylane/devices/tests/pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
 markers =
     skip_unsupported: skip a test if it uses an operation unsupported on a device
-addopts = --benchmark-disable

requirements-ci.txt

@@ -13,3 +13,4 @@ matplotlib
 requests
 rich
 tomli # Drop once minimum Python version is 3.11
+pandas