Reformat python code (#905)

# Pull Request

## Title

Updates most of the python code for updated syntax after #904 

---

## Description

- Reformats the code in bulks with `pyupgrade`
- Apply some additional autoflake and isort fixes for the no longer
necessary imports.
- Fixup a few places that need to use `__futures__ import annotations`
for forward references in type hints (and run pyupgrade again to remove
the previous quoted string versions).
- Adjusts a few docstrings, but not all of the `Optional` and `Union`
ones (due to difficulties with regex and matching braces).
- Closes #749 

---

## Type of Change

- 🔄 Refactor

---

## Testing

Usual CI

---

## Additional Notes (optional)

Should be merged after #904  

---

.pre-commit-config.yaml

@@ -36,14 +36,13 @@ repos:
         args: [-t, doc/mit-license.tmpl, -E, .py, .sh, .ps1, .sql, .cmd, -x, mlos_bench/setup.py, mlos_core/setup.py, mlos_viz/setup.py]
         require_serial: true
         stages: [pre-commit, manual]
-# TODO:
-#-   repo: https://github.com/asottile/pyupgrade
-#    rev: v3.19.1
-#    hooks:
-#    -   id: pyupgrade
-#        args: [--py310-plus]
-#        require_serial: true
-#        stages: [pre-commit, manual]
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.19.1
+    hooks:
+    -   id: pyupgrade
+        args: [--py310-plus]
+        require_serial: true
+        stages: [pre-commit, manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.13.2
     hooks:

conftest.py

@@ -35,13 +35,11 @@ def pytest_configure(config: pytest.Config) -> None:
                 # Only warn once.
                 warn(
                     UserWarning(
-                        (
-                            "DISPLAY environment variable is set, "
-                            "which can cause problems in some setups (e.g. WSL). "
-                            "Adjusting matplotlib backend to "
-                            f"""'{matplotlib.rcParams["backend"]}' """
-                            "to compensate."
-                        )
+                        "DISPLAY environment variable is set, "
+                        "which can cause problems in some setups (e.g. WSL). "
+                        "Adjusting matplotlib backend to "
+                        f"""'{matplotlib.rcParams["backend"]}' """
+                        "to compensate."
                     )
                 )
         except ImportError:

doc/source/conf.py

@@ -22,7 +22,6 @@
 import os
 import sys
 from logging import warning
-from typing import Dict, Optional, Tuple
 
 from docutils.nodes import Element
 from intersphinx_registry import get_intersphinx_mapping
@@ -92,7 +91,7 @@
 napoleon_custom_sections = None
 
 _base_path = os.path.abspath(os.path.join(__file__, "../../.."))
-_path_cache: Dict[str, bool] = {}
+_path_cache: dict[str, bool] = {}
 
 
 def _check_path(path: str) -> bool:
@@ -105,7 +104,7 @@ def _check_path(path: str) -> bool:
     return result
 
 
-def linkcode_resolve(domain: str, info: Dict[str, str]):
+def linkcode_resolve(domain: str, info: dict[str, str]):
     """Linkcode extension override to link to the source code on GitHub."""
     if domain != "py":
         return None
@@ -156,7 +155,7 @@ def is_on_github_actions():
 
 # Type alias resolution map
 # (original, refname) -> new
-CUSTOM_REF_TYPE_MAP: Dict[Tuple[str, str], str] = {
+CUSTOM_REF_TYPE_MAP: dict[tuple[str, str], str] = {
     # Internal typevars and aliases:
     ("BaseTypeVar", "class"): "data",
     ("ConcreteOptimizer", "class"): "data",
@@ -180,7 +179,7 @@ def resolve_type_aliases(
     env: BuildEnvironment,
     node: pending_xref,
     contnode: Element,
-) -> Optional[Element]:
+) -> Element | None:
     """Resolve :class: references to our type aliases as :attr: instead."""
     if node["refdomain"] != "py":
         return None

mlos_bench/mlos_bench/config/environments/apps/fio/scripts/local/process_fio_results.py

@@ -8,12 +8,13 @@
 import argparse
 import itertools
 import json
-from typing import Any, Iterator, Tuple
+from collections.abc import Iterator
+from typing import Any
 
 import pandas
 
 
-def _flat_dict(data: Any, path: str) -> Iterator[Tuple[str, Any]]:
+def _flat_dict(data: Any, path: str) -> Iterator[tuple[str, Any]]:
     """Flatten every dict in the hierarchy and rename the keys with the dict path."""
     if isinstance(data, dict):
         for key, val in data.items():
@@ -24,7 +25,7 @@ def _flat_dict(data: Any, path: str) -> Iterator[Tuple[str, Any]]:
 
 def _main(input_file: str, output_file: str, prefix: str) -> None:
     """Convert FIO read data from JSON to tall CSV."""
-    with open(input_file, mode="r", encoding="utf-8") as fh_input:
+    with open(input_file, encoding="utf-8") as fh_input:
         json_data = json.load(fh_input)
 
     data = list(

mlos_bench/mlos_bench/config/environments/apps/redis/scripts/local/generate_redis_config.py

@@ -15,8 +15,8 @@
 
 def _main(fname_input: str, fname_output: str) -> None:
     with (
-        open(fname_input, "rt", encoding="utf-8") as fh_tunables,
-        open(fname_output, "wt", encoding="utf-8", newline="") as fh_config,
+        open(fname_input, encoding="utf-8") as fh_tunables,
+        open(fname_output, "w", encoding="utf-8", newline="") as fh_config,
     ):
         for key, val in json.load(fh_tunables).items():
             line = f"{key} {val}"

mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/create_new_grub_cfg.py

@@ -17,7 +17,7 @@
 
 def _write_config() -> None:
     with (
-        open(JSON_CONFIG_FILE, "r", encoding="UTF-8") as fh_json,
+        open(JSON_CONFIG_FILE, encoding="UTF-8") as fh_json,
         open(NEW_CFG, "w", encoding="UTF-8") as fh_config,
     ):
         for key, val in json.load(fh_json).items():

mlos_bench/mlos_bench/config/environments/os/linux/boot/scripts/local/generate_grub_config.py

@@ -15,8 +15,8 @@
 
 def _main(fname_input: str, fname_output: str) -> None:
     with (
-        open(fname_input, "rt", encoding="utf-8") as fh_tunables,
-        open(fname_output, "wt", encoding="utf-8", newline="") as fh_config,
+        open(fname_input, encoding="utf-8") as fh_tunables,
+        open(fname_output, "w", encoding="utf-8", newline="") as fh_config,
     ):
         for key, val in json.load(fh_tunables).items():
             line = f'GRUB_CMDLINE_LINUX_DEFAULT="${{GRUB_CMDLINE_LINUX_DEFAULT}} {key}={val}"'

mlos_bench/mlos_bench/config/environments/os/linux/runtime/scripts/local/generate_kernel_config_script.py

@@ -18,13 +18,13 @@
 
 def _main(fname_input: str, fname_meta: str, fname_output: str) -> None:
 
-    with open(fname_input, "rt", encoding="utf-8") as fh_tunables:
+    with open(fname_input, encoding="utf-8") as fh_tunables:
         tunables_data = json.load(fh_tunables)
 
-    with open(fname_meta, "rt", encoding="utf-8") as fh_meta:
+    with open(fname_meta, encoding="utf-8") as fh_meta:
         tunables_meta = json.load(fh_meta)
 
-    with open(fname_output, "wt", encoding="utf-8", newline="") as fh_config:
+    with open(fname_output, "w", encoding="utf-8", newline="") as fh_config:
         for key, val in tunables_data.items():
             meta = tunables_meta.get(key, {})
             name_prefix = meta.get("name_prefix", "")

mlos_bench/mlos_bench/config/schemas/config_schemas.py

@@ -23,9 +23,9 @@
 
 import json  # schema files are pure json - no comments
 import logging
+from collections.abc import Iterator, Mapping
 from enum import Enum
 from os import environ, path, walk
-from typing import Dict, Iterator, Mapping
 
 import jsonschema
 from referencing import Registry, Resource
@@ -70,7 +70,7 @@ class SchemaStore(Mapping):
     """
 
     # A class member mapping of schema id to schema object.
-    _SCHEMA_STORE: Dict[str, dict] = {}
+    _SCHEMA_STORE: dict[str, dict] = {}
     _REGISTRY: Registry = Registry()
 
     def __len__(self) -> int:
@@ -99,7 +99,7 @@ def _load_schemas(cls) -> None:
                 file_path = path_join(root, file_name)
                 if path.getsize(file_path) == 0:
                     continue
-                with open(file_path, mode="r", encoding="utf-8") as schema_file:
+                with open(file_path, encoding="utf-8") as schema_file:
                     schema = json.load(schema_file)
                     cls._SCHEMA_STORE[file_path] = schema
                     # Let the schema be referenced by its id as well.

mlos_bench/mlos_bench/dict_templater.py

@@ -8,47 +8,47 @@
 
 from copy import deepcopy
 from string import Template
-from typing import Any, Dict, Optional
+from typing import Any
 
 from mlos_bench.os_environ import environ
 
 
 class DictTemplater:  # pylint: disable=too-few-public-methods
     """Simple class to help with nested dictionary ``$var`` templating."""
 
-    def __init__(self, source_dict: Dict[str, Any]):
+    def __init__(self, source_dict: dict[str, Any]):
         """
         Initialize the templater.
 
         Parameters
         ----------
-        source_dict : Dict[str, Any]
+        source_dict : dict[str, Any]
             The template dict to use for source variables.
         """
         # A copy of the initial data structure we were given with templates intact.
         self._template_dict = deepcopy(source_dict)
         # The source/target dictionary to expand.
-        self._dict: Dict[str, Any] = {}
+        self._dict: dict[str, Any] = {}
 
     def expand_vars(
         self,
         *,
-        extra_source_dict: Optional[Dict[str, Any]] = None,
+        extra_source_dict: dict[str, Any] | None = None,
         use_os_env: bool = False,
-    ) -> Dict[str, Any]:
+    ) -> dict[str, Any]:
         """
         Expand the template variables in the destination dictionary.
 
         Parameters
         ----------
-        extra_source_dict : Dict[str, Any]
+        extra_source_dict : dict[str, Any]
             An optional extra source dictionary to use for expansion.
         use_os_env : bool
             Whether to use the os environment variables a final fallback for expansion.
 
         Returns
         -------
-        Dict[str, Any]
+        dict[str, Any]
             The expanded dictionary.
 
         Raises
@@ -64,7 +64,7 @@ def expand_vars(
     def _expand_vars(
         self,
         value: Any,
-        extra_source_dict: Optional[Dict[str, Any]],
+        extra_source_dict: dict[str, Any] | None,
         use_os_env: bool,
     ) -> Any:
         """Recursively expand ``$var`` strings in the currently operating dictionary."""