Merge pull request PowerDNS#14935 from omoerbeek/rec-regr-test-markers

rec: regression test markers
webfutureiorepo · Dec 16, 2024 · bd7b45a · bd7b45a
2 parents 7ce6463 + a8ba292
commit bd7b45a
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 3 deletions.
diff --git a/regression-tests.recursor-dnssec/pytest.ini b/regression-tests.recursor-dnssec/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+markers = 
+    external: uses external web servers
+    unreliable_on_gh: test is fine when run locally, but shows issues on GitHub
diff --git a/regression-tests.recursor-dnssec/runtests b/regression-tests.recursor-dnssec/runtests
@@ -61,9 +61,13 @@ if ! "$PDNSRECURSOR" --version 2>&1 | grep Features | grep -q dnstap-framestream
   export NODNSTAPTESTS=1
 fi
 
+# Run with -m 'not external' to skip test that require external connectivity
+# Run with -m 'not unreliable_on_gh' to skip tests that are unreliable on GitHUb
+# Run with -m 'not (external or unreliable_on_gh)' to skip both categories
+
 # LIBFAKETIME is only added to LD_PRELOAD by the pyton code when needed
 if [ "${LIBASAN}" != "" -o "${LIBAUTHBIND}" != "" ]; then
-LD_PRELOAD="${LIBASAN} ${LIBAUTHBIND}" pytest --ignore=test_WellKnown.py --junitxml=pytest.xml $@
+LD_PRELOAD="${LIBASAN} ${LIBAUTHBIND}" pytest --junitxml=pytest.xml "$@"
 else
-pytest --ignore=test_WellKnown.py  --junitxml=pytest.xml $@
+pytest --junitxml=pytest.xml "$@"
 fi
diff --git a/regression-tests.recursor-dnssec/test_Chain.py b/regression-tests.recursor-dnssec/test_Chain.py
@@ -1,3 +1,4 @@
+import pytest
 import dns
 import os
 import time
@@ -23,6 +24,7 @@ class ChainTest(RecursorTest):
     api-key=%s
 """ % (_wsPort, _wsPassword, _apiKey)
 
+    @pytest.mark.unreliable_on_gh
     def testBasic(self):
         """
         Tests the case of #14624. Sending many equal requests could lead to ServFail because of clashing

diff --git a/regression-tests.recursor-dnssec/test_ExtendedErrors.py b/regression-tests.recursor-dnssec/test_ExtendedErrors.py
@@ -83,6 +83,7 @@ def generateRecursorConfig(cls, confdir):
 
         super(ExtendedErrorsTest, cls).generateRecursorConfig(confdir)
 
+    @pytest.mark.external
     def testNotIncepted(self):
         qname = 'signotincepted.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -96,6 +97,7 @@ def testNotIncepted(self):
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(8, b''))
 
+    @pytest.mark.external
     def testExpired(self):
         qname = 'sigexpired.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -109,6 +111,7 @@ def testExpired(self):
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(7, b''))
 
+    @pytest.mark.external
     def testAllExpired(self):
         qname = 'servfail.nl.'
         query = dns.message.make_query(qname, 'AAAA', want_dnssec=True)
@@ -122,6 +125,7 @@ def testAllExpired(self):
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(6, b''))
 
+    @pytest.mark.external
     def testBogus(self):
         qname = 'bogussig.ok.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -135,6 +139,7 @@ def testBogus(self):
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(6, b''))
 
+    @pytest.mark.external
     def testMissingRRSIG(self):
         qname = 'brokendnssec.net.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -236,6 +241,7 @@ def tearDownClass(cls):
     def generateRecursorConfig(cls, confdir):
         super(NoExtendedErrorsTest, cls).generateRecursorConfig(confdir)
 
+    @pytest.mark.external
     def testNotIncepted(self):
         qname = 'signotincepted.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)

diff --git a/regression-tests.recursor-dnssec/test_SimpleDoT.py b/regression-tests.recursor-dnssec/test_SimpleDoT.py
@@ -1,3 +1,4 @@
+import pytest
 import dns
 import os
 import subprocess
@@ -30,6 +31,7 @@ def setUpClass(cls):
         cls.generateRecursorConfig(confdir)
         cls.startRecursor(confdir, cls._recursorPort)
 
+    @pytest.mark.external
     def testTXT(self):
         expected = dns.rrset.from_text('dot-test-target.powerdns.org.', 0, dns.rdataclass.IN, 'TXT', 'https://github.com/PowerDNS/pdns/pull/12825')
         query = dns.message.make_query('dot-test-target.powerdns.org', 'TXT', want_dnssec=True)

diff --git a/regression-tests.recursor-dnssec/test_SimpleForwardOverDoT.py b/regression-tests.recursor-dnssec/test_SimpleForwardOverDoT.py
@@ -1,3 +1,4 @@
+import pytest
 import dns
 import os
 import subprocess
@@ -27,6 +28,7 @@ def setUpClass(cls):
         cls.generateRecursorConfig(confdir)
         cls.startRecursor(confdir, cls._recursorPort)
 
+    @pytest.mark.external
     def testA(self):
         expected = dns.rrset.from_text('dns.google.', 0, dns.rdataclass.IN, 'A', '8.8.8.8', '8.8.4.4')
         query = dns.message.make_query('dns.google', 'A', want_dnssec=True)

diff --git a/regression-tests.recursor-dnssec/test_WellKnown.py b/regression-tests.recursor-dnssec/test_WellKnown.py
@@ -1,7 +1,9 @@
+import pytest
 import dns
 from recursortests import RecursorTest
 
-class TestWellKnown(RecursorTest):
+@pytest.mark.external
+class WellKnownTest(RecursorTest):
     _auths_zones = None
     _confdir = 'WellKnown'
     _roothints = None