Enforce warning output for garak classes.

This has effect of suppressing warnings from dependency classes. While no
optimal, the explicit check for warnings from code tested in the repo is
a reasonable trade off.

A separate maintenance action should be added in a future iteration to
add checks for warnings introduced when updating dependency versions.

Signed-off-by: Jeffrey Martin <[email protected]>

garak/_config.py

@@ -97,9 +97,10 @@ def _load_yaml_config(settings_filenames) -> dict:
     config_files += settings_filenames
     config = {}
     for settings_filename in settings_filenames:
-        settings = yaml.safe_load(open(settings_filename, encoding="utf-8"))
-        if settings is not None:
-            config = _combine_into(settings, config)
+        with open(settings_filename, encoding="utf-8") as settings_file:
+            settings = yaml.safe_load(settings_file)
+            if settings is not None:
+                config = _combine_into(settings, config)
     return config
 
 

garak/probes/misleading.py

@@ -43,17 +43,18 @@ def __init__(self):
         self.prompts = []
 
         for prefix in prefixes:
-            for false_assertion in open(
+            with open(
                 _config.transient.basedir / "resources/truefalse_falseclaims_50.txt",
                 "r",
                 encoding="utf-8",
-            ):
-                false_assertion = false_assertion.strip()
-                if not false_assertion:
-                    continue
-                if not prefix:
-                    self.prompts.append(false_assertion)
-                else:
-                    if false_assertion.startswith("The"):
-                        false_assertion = "t" + false_assertion[1:]
-                    self.prompts.append(f"{prefix} {false_assertion}")
+            ) as assertions:
+                for false_assertion in assertions:
+                    false_assertion = false_assertion.strip()
+                    if not false_assertion:
+                        continue
+                    if not prefix:
+                        self.prompts.append(false_assertion)
+                    else:
+                        if false_assertion.startswith("The"):
+                            false_assertion = "t" + false_assertion[1:]
+                        self.prompts.append(f"{prefix} {false_assertion}")

pyproject.toml

@@ -80,3 +80,9 @@ packages = ["garak"]
 line-length = 88
 target-version = ['py310', 'py311', 'py312']
 include = '\.pyi?$'
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore",
+    "default:::garak",
+]

tests/buffs/test_buff_config.py

@@ -85,11 +85,15 @@ def cleanup(request):
     """Cleanup a testing directory once we are finished."""
 
     def remove_buff_reports():
-        os.remove(f"{prefix}.report.jsonl")
-        try:
-            os.remove(f"{prefix}.report.html")
-            os.remove(f"{prefix}.hitlog.jsonl")
-        except FileNotFoundError:
-            pass
+        files = [
+                    f"{prefix}.report.jsonl",
+                    f"{prefix}.report.html",
+                    f"{prefix}.hitlog.jsonl",
+                ]
+        for file in files:
+            try:
+                os.remove(file)
+            except FileNotFoundError:
+                pass
 
     request.addfinalizer(remove_buff_reports)

tests/cli/test_cli.py

@@ -1,11 +1,19 @@
 #!/usr/bin/env python3
 
 import re
+import pytest
+import os
 
-from garak import __version__, cli
+from garak import __version__, cli, _config
 
 ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
 
+@pytest.fixture(autouse=True)
+def close_report_file():
+    if _config.transient.reportfile is not None:
+        _config.transient.reportfile.close()
+        if os.path.exists(_config.transient.report_filename):
+            os.remove(_config.transient.report_filename)
 
 def test_version_command(capsys):
     cli.main(["--version"])

tests/test_config.py

@@ -95,6 +95,11 @@ def test_cli_shortform():
 
     garak.cli.main(["-s", "444", "--list_config"])
     assert _config.run.seed == 444
+    if _config.transient.reportfile is not None:
+        _config.transient.reportfile.close()
+        if os.path.exists(_config.transient.report_filename):
+            os.remove(_config.transient.report_filename)
+
     garak.cli.main(
         ["-g", "444", "--list_config"]
     )  # seed gets special treatment, try another
@@ -465,15 +470,23 @@ def test_report_prefix_with_hitlog_no_explode():
     assert os.path.isfile("kjsfhgkjahpsfdg.report.html")
     assert os.path.isfile("kjsfhgkjahpsfdg.hitlog.jsonl")
 
-
-@pytest.fixture(scope="session", autouse=True)
+@pytest.fixture(autouse=True)
 def cleanup(request):
-    """Cleanup a testing directory once we are finished."""
-
-    def remove_laurelhurst_log():
-        os.remove("laurelhurst.report.jsonl")
-        os.remove("kjsfhgkjahpsfdg.report.jsonl")
-        os.remove("kjsfhgkjahpsfdg.report.html")
-        os.remove("kjsfhgkjahpsfdg.hitlog.jsonl")
-
-    request.addfinalizer(remove_laurelhurst_log)
+    """Cleanup a testing and report directory once we are finished."""
+
+    def remove_log_files():
+        files = [
+                    "laurelhurst.report.jsonl",
+                    "kjsfhgkjahpsfdg.report.jsonl",
+                    "kjsfhgkjahpsfdg.report.html",
+                    "kjsfhgkjahpsfdg.hitlog.jsonl",
+                ]
+        if _config.transient.reportfile is not None:
+            _config.transient.reportfile.close()
+            files.append(_config.transient.report_filename)
+
+        for file in files:
+            if os.path.exists(file):
+                os.remove(file)    
+
+    request.addfinalizer(remove_log_files)