Issue #28 individual process tests: skip all processing by default

Implements a dummy  SkippingRunner, which is now picked by default

src/openeo_test_suite/lib/process_runner/skip.py

@@ -0,0 +1,13 @@
+from typing import Dict, List
+
+import pytest
+
+from openeo_test_suite.lib.process_runner.base import ProcessTestRunner
+
+
+class SkippingRunner(ProcessTestRunner):
+    def list_processes(self) -> List[Dict]:
+        pytest.skip(f"SkippingRunner: No processes")
+
+    def execute(self, id, arguments):
+        pytest.skip(f"SkippingRunner: skip executing process {id}")

src/openeo_test_suite/tests/processes/README.md

@@ -9,21 +9,34 @@
 
 ## Individual Process Testing
 
-### Examples
+### Usage examples
 
-- `pytest --openeo-backend-url=https://openeo.cloud --processes=min,max`
-- `pytest --runner=vito --process-levels=L1,L2,L2A`
-- `pytest --runner=dask`
-- `pytest src/openeo_test_suite/tests/processes/processing/test_example.py --runner=dask`
+Specify test path `src/openeo_test_suite/tests/processes/processing` to only run individual process tests.
 
-### Parameters
 
-Specify `src/openeo_test_suite/tests/processes/processing/test_example.py` to only run individual process tests.
+```bash
+# Basic default behavior: run all individual process tests,
+# but with the default runner (skip), so no tests will actually be executed.
+pytest src/openeo_test_suite/tests/processes
+
+# Run tests for a subset of processes with the HTTP runner
+# against the openEO Platform backend at openeo.cloud
+pytest --runner=http --openeo-backend-url=openeo.cloud --processes=min,max src/openeo_test_suite/tests/processes/processing
+
+# Run tests for a subset of processes with the VITO runner
+pytest --runner=vito --process-levels=L1,L2,L2A src/openeo_test_suite/tests/processes/processing
+
+# Run all individual process tests with the Dask runner
+pytest --runner=dask src/openeo_test_suite/tests/processes
+```
+
+### Parameters
 
 - `--runner`: The execution engine. One of:
-  - `vito` (needs <https://github.com/Open-EO/openeo-python-driver> being installed)
-  - `dask` (needs optional Dask dependencies)
-  - `http` (**default**, requires `--openeo-backend-url` to be set)
+  - `skip` (**default**) skip all individual process tests
+  - `vito` (requires [openeo_driver](https://github.com/Open-EO/openeo-python-driver) package being installed in test environment)
+  - `dask` (requires [openeo_processes_dask](https://github.com/Open-EO/openeo-processes-dask) package being installed in test environment)
+  - `http` (requires `--openeo-backend-url` to be set)
 - `--process-levels`: All [process profiles](https://openeo.org/documentation/1.0/developers/profiles/processes.html) to test against, separated by comma. You need to list all levels explicitly, e.g., L2 does **not** include L1 automatically. Example: `L1,L2,L2A`. By default tests against all processes.
 - `--processes`: A list of processes to test against, separated by comma. Example: `apply,reduce_dimension`. By default tests against all processes.
 - `--experimental`: Enables tests for experimental processes. By default experimental processes will be skipped.

src/openeo_test_suite/tests/processes/processing/conftest.py

@@ -21,7 +21,7 @@ def runner(request) -> str:
     elif "OPENEO_RUNNER" in os.environ:
         runner = os.environ["OPENEO_RUNNER"]
     else:
-        runner = "http"
+        runner = "skip"
 
     _log.info(f"Using runner {runner!r}")
 
@@ -41,6 +41,9 @@ def auto_authenticate() -> bool:
 def connection(
     request, runner: str, auto_authenticate: bool, pytestconfig
 ) -> ProcessTestRunner:
+    # TODO: this fixture override changes the return type of the original `connection` fixture,
+    #       which might lead to problems due to broken assumptions
+
     if runner == "dask":
         from openeo_test_suite.lib.process_runner.dask import Dask
 
@@ -49,7 +52,11 @@ def connection(
         from openeo_test_suite.lib.process_runner.vito import Vito
 
         return Vito()
-    else:
+    elif runner == "skip":
+        from openeo_test_suite.lib.process_runner.skip import SkippingRunner
+
+        return SkippingRunner()
+    elif runner == "http":
         from openeo_test_suite.lib.process_runner.http import Http
 
         backend_url = get_backend_url(request.config, required=True)
@@ -62,3 +69,6 @@ def connection(
                 con.authenticate_oidc()
 
         return Http(con)
+
+    else:
+        raise ValueError(f"Unknown runner {runner!r}")

src/openeo_test_suite/tests/processes/processing/test_example.py

@@ -55,9 +55,6 @@ def test_process(
     experimental,
     skipper,
 ):
-    if skip_experimental and experimental:
-        pytest.skip("Skipping experimental process {}".format(id))
-
     skipper.skip_if_unmatching_process_level(level)
     if len(processes) > 0 and id not in processes:
         pytest.skip(
@@ -67,6 +64,9 @@ def test_process(
     # check whether the process is available
     skipper.skip_if_unsupported_process([id])
 
+    if skip_experimental and experimental:
+        pytest.skip("Skipping experimental process {}".format(id))
+
     # check whether any additionally required processes are available
     if "required" in example:
         skipper.skip_if_unsupported_process(example["required"])