From f5c0894b7bfd8c0b3811d624fc2f2969d5af9e8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= <m.risse@fz-juelich.de>
Date: Tue, 19 Mar 2024 10:17:03 +0100
Subject: [PATCH] Run the test suite in a random order

---
 .github/workflows/testing-all-oses.yml |  4 ++--
 tests/_test_mswms/test_wms.py          | 11 ++++-------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/testing-all-oses.yml b/.github/workflows/testing-all-oses.yml
index 2083af10a..92f38804c 100644
--- a/.github/workflows/testing-all-oses.yml
+++ b/.github/workflows/testing-all-oses.yml
@@ -19,7 +19,6 @@ jobs:
       fail-fast: false
       matrix:
         os: ["macos-13", "macos-14", "ubuntu-latest"]
-        order: ["normal", "reverse"]
     steps:
     - uses: actions/checkout@v4
     - name: Build requirements.txt file
@@ -29,6 +28,7 @@ jobs:
           sed -e "s/menuinst.*//" |
           sed -e "s/.*://" > requirements.tmp.txt
         cat requirements.d/development.txt >> requirements.tmp.txt
+        echo "pytest-randomly" >> requirements.tmp.txt
         sed -e '/^$/d' -e '/^#.*$/d' requirements.tmp.txt > requirements.txt
         rm requirements.tmp.txt
         cat requirements.txt
@@ -50,4 +50,4 @@ jobs:
       # TODO: fix those tests and drop the ignores
       run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib
         --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py
-        ${{ (matrix.order == 'normal' && ' ') || (matrix.order == 'reverse' && '--reverse') }} tests
+        tests
diff --git a/tests/_test_mswms/test_wms.py b/tests/_test_mswms/test_wms.py
index ccce3301b..2cb33103c 100644
--- a/tests/_test_mswms/test_wms.py
+++ b/tests/_test_mswms/test_wms.py
@@ -25,7 +25,6 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 """
-import sys
 import os
 from shutil import move
 
@@ -388,11 +387,11 @@ def test_import_error(self):
         assert mslib.mswms.wms.mswms_settings.__file__ is not None
         assert mslib.mswms.wms.mswms_auth.__file__ is not None
 
-    @pytest.mark.skipif(
-        sys.platform == "darwin",
-        reason="""\
+    @pytest.mark.skip("""\
 There is a race condition between modifying with ncap2 and asserting that the file changed where the server might not
 see the change before the request is made, which leads to a failure of the following assert.
+
+This test fails on macOS 14 and can also fail on Linux when the pytest test order is randomized.
 """.strip(),
     )
     def test_files_changed(self):
@@ -436,9 +435,7 @@ def do_test():
             "data_access", new=watch_access):
             do_test()
 
-    @pytest.mark.skipif(
-        sys.platform == "darwin",
-        reason="""\
+    @pytest.mark.skip("""\
 This test changes global variables (e.g. DOCS_LOCATION) which can affect other tests depending on test order
 (e.g. tests/_test_mswms/test_mss_plot_driver.py::Test_VSec::test_VS_gallery_template fails consistently in reverse order
 on macOS 14).