Merge pull request #161 from CasperWA/use_simple_caching

Add caching using `CacheControl`:
https://cachecontrol.readthedocs.io/

Storing an object-store-like cache using `lockfile`.
This should ensure Thread-safe caching.

Avoid caching `localhost` - important for development.

---

Better integrate `flake8`.

.github/workflows/ci.yml

@@ -17,15 +17,15 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install -U pip
         pip install flake8
 
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 . --count --statistics
 
   pre-commit:
 
@@ -46,7 +46,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install -U pip
         pip install -U setuptools
         pip install pre-commit
 
@@ -72,7 +72,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install -U pip
         pip install -U setuptools
         pip install -e .[testing]
 

optimade_client/informational.py

@@ -1,13 +1,14 @@
 import logging
 import os
 from pathlib import Path
+import shutil
 from typing import Union
 from urllib.parse import urlencode
 
 import ipywidgets as ipw
 
-from optimade_client.logger import LOGGER, WIDGET_HANDLER, REPORT_HANDLER
-from optimade_client.utils import __optimade_version__, ButtonStyle
+from optimade_client.logger import LOG_DIR, LOGGER, REPORT_HANDLER, WIDGET_HANDLER
+from optimade_client.utils import __optimade_version__, ButtonStyle, CACHE_DIR
 
 
 IMG_DIR = Path(__file__).parent.joinpath("img")
@@ -244,16 +245,54 @@ def __init__(self, **kwargs):
             description="Show DEBUG messages",
             disabled=False,
             indent=False,
+            width="auto",
+            height="auto",
+        )
+        self.clear_cache = ipw.Button(
+            description="Clear cache",
+            disabled=False,
+            tooltip="Clear cached responses (not logs)",
+            icon="cube",
+            layout={
+                "visibility": "visible" if self._debug else "hidden",
+                "width": "auto",
+            },
+        )
+        self.clear_logs = ipw.Button(
+            description="Clear logs",
+            disabled=False,
+            tooltip="Clear all log history",
+            icon="edit",
+            layout={
+                "visibility": "visible" if self._debug else "hidden",
+                "width": "auto",
+            },
         )
         self.log_output = WIDGET_HANDLER.get_widget()
         super().__init__(
-            children=(ipw.VBox(children=(self.toggle_debug, self.log_output)),),
+            children=(
+                ipw.VBox(
+                    children=(
+                        ipw.HBox(
+                            children=(
+                                self.toggle_debug,
+                                self.clear_cache,
+                                self.clear_logs,
+                            ),
+                            layout={"height": "auto", "width": "auto"},
+                        ),
+                        self.log_output,
+                    )
+                ),
+            ),
             **kwargs,
         )
         self.set_title(0, "Log")
-        self.selected_index = None
+        self.selected_index = 0 if self._debug else None
 
         self.toggle_debug.observe(self._toggle_debug_logging, names="value")
+        self.clear_cache.on_click(self._clear_cache)
+        self.clear_logs.on_click(self._clear_logs)
 
     def freeze(self):
         """Disable widget"""
@@ -272,16 +311,68 @@ def reset(self):
         self.toggle_debug.disabled = False
         self.log_output.reset()
 
-    @staticmethod
-    def _toggle_debug_logging(change: dict):
+    def _toggle_debug_logging(self, change: dict):
         """Set logging level depending on toggle button"""
         if change["new"]:
             # Set logging level DEBUG
             WIDGET_HANDLER.setLevel(logging.DEBUG)
             LOGGER.info("Set log output in widget to level DEBUG")
             LOGGER.debug("This should now be shown")
+
+            # Show debug buttons
+            self.clear_cache.layout.visibility = "visible"
+            self.clear_logs.layout.visibility = "visible"
         else:
             # Set logging level to INFO
             WIDGET_HANDLER.setLevel(logging.INFO)
             LOGGER.info("Set log output in widget to level INFO")
             LOGGER.debug("This should now NOT be shown")
+
+            # Hide debug buttons
+            self.clear_cache.layout.visibility = "hidden"
+            self.clear_logs.layout.visibility = "hidden"
+
+    @staticmethod
+    def _clear_cache(_):
+        """Clear cached responses (not logs)"""
+        if str(LOG_DIR).startswith(str(CACHE_DIR)):
+            log_sub_dir = list(Path(str(LOG_DIR)[len(f"{CACHE_DIR}/") :]).parts)
+
+        LOGGER.debug(
+            "Cache dir: %s - Log dir: %s - Log sub dir parts: %s",
+            CACHE_DIR,
+            LOG_DIR,
+            log_sub_dir,
+        )
+
+        for dirpath, dirnames, filenames in os.walk(CACHE_DIR):
+            log_dir_part = log_sub_dir.pop(0) if log_sub_dir else ""
+            if not log_sub_dir:
+                LOGGER.debug(
+                    "No more log sub directory parts. Removing %r from dirnames list.",
+                    log_dir_part,
+                )
+                dirnames.remove(log_dir_part)
+
+            for directory in list(dirnames):
+                if directory == log_dir_part:
+                    continue
+                LOGGER.debug(
+                    "Removing folder: %s", Path(dirpath).joinpath(directory).resolve()
+                )
+                shutil.rmtree(
+                    Path(dirpath).joinpath(directory).resolve(), ignore_errors=True
+                )
+                dirnames.remove(directory)
+            for filename in filenames:
+                LOGGER.debug(
+                    "Removing file: %s", Path(dirpath).joinpath(filename).resolve()
+                )
+                os.remove(Path(dirpath).joinpath(filename).resolve())
+        CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+    @staticmethod
+    def _clear_logs(_):
+        """Clear all logs"""
+        shutil.rmtree(LOG_DIR, ignore_errors=True)
+        LOG_DIR.mkdir(parents=True, exist_ok=True)

optimade_client/query_filter.py

@@ -1,7 +1,7 @@
 from typing import Union
-import requests
 import traitlets
 import ipywidgets as ipw
+import requests
 
 try:
     from simplejson import JSONDecodeError
@@ -15,16 +15,18 @@
 from optimade_client.exceptions import BadResource, QueryError
 from optimade_client.logger import LOGGER
 from optimade_client.subwidgets import (
-    StructureDropdown,
     FilterTabs,
     ResultsPageChooser,
+    StructureDropdown,
 )
 from optimade_client.utils import (
     ButtonStyle,
-    perform_optimade_query,
+    check_entry_properties,
     handle_errors,
+    ordered_query_url,
+    perform_optimade_query,
+    SESSION,
     TIMEOUT_SECONDS,
-    check_entry_properties,
 )
 
 
@@ -372,7 +374,11 @@ def _query(self, link: str = None) -> dict:
         # If a complete link is provided, use it straight up
         if link is not None:
             try:
-                response = requests.get(link, timeout=TIMEOUT_SECONDS).json()
+                link = ordered_query_url(link)
+                response = SESSION.get(link, timeout=TIMEOUT_SECONDS)
+                if response.from_cache:
+                    LOGGER.debug("Request to %s was taken from cache !", link)
+                response = response.json()
             except (
                 requests.exceptions.ConnectTimeout,
                 requests.exceptions.ConnectionError,

optimade_client/subwidgets/__init__.py

@@ -1,17 +1,17 @@
 # pylint: disable=undefined-variable
-from .filter_inputs import *
-from .multi_checkbox import *
-from .output_summary import *
-from .periodic_table import *
-from .provider_database import *
-from .results import *
+from .filter_inputs import *  # noqa: F403
+from .multi_checkbox import *  # noqa: F403
+from .output_summary import *  # noqa: F403
+from .periodic_table import *  # noqa: F403
+from .provider_database import *  # noqa: F403
+from .results import *  # noqa: F403
 
 
 __all__ = (
-    filter_inputs.__all__  # noqa
-    + multi_checkbox.__all__  # noqa
-    + output_summary.__all__  # noqa
-    + periodic_table.__all__  # noqa
-    + provider_database.__all__  # noqa
-    + results.__all__  # noqa
+    filter_inputs.__all__  # noqa: F405
+    + multi_checkbox.__all__  # noqa: F405
+    + output_summary.__all__  # noqa: F405
+    + periodic_table.__all__  # noqa: F405
+    + provider_database.__all__  # noqa: F405
+    + results.__all__  # noqa: F405
 )

optimade_client/subwidgets/provider_database.py

@@ -11,20 +11,22 @@
 import requests
 import traitlets
 
-from optimade.models import LinksResourceAttributes, LinksResource
+from optimade.models import LinksResource, LinksResourceAttributes
 from optimade.models.links import LinkType
 
-from optimade_client.exceptions import QueryError, OptimadeClientError
+from optimade_client.exceptions import OptimadeClientError, QueryError
 from optimade_client.logger import LOGGER
 from optimade_client.subwidgets.results import ResultsPageChooser
 from optimade_client.utils import (
     get_list_of_valid_providers,
     get_versioned_base_url,
     handle_errors,
+    ordered_query_url,
     perform_optimade_query,
-    validate_api_version,
+    SESSION,
     TIMEOUT_SECONDS,
     update_old_links_resources,
+    validate_api_version,
 )
 
 
@@ -453,7 +455,11 @@ def _query(  # pylint: disable=too-many-locals,too-many-branches,too-many-statem
                         f"?{parsed_query}"
                     )
 
-                response = requests.get(link, timeout=TIMEOUT_SECONDS).json()
+                link = ordered_query_url(link)
+                response = SESSION.get(link, timeout=TIMEOUT_SECONDS)
+                if response.from_cache:
+                    LOGGER.debug("Request to %s was taken from cache !", link)
+                response = response.json()
             except (
                 requests.exceptions.ConnectTimeout,
                 requests.exceptions.ConnectionError,
@@ -641,13 +647,13 @@ def _on_database_change(self, change):
     def _update_provider(self):
         """Update provider summary"""
         html_text = f"""<strong style="line-height:1;{self.text_style}">{getattr(self.provider, 'name', 'Provider')}</strong>
-        <p style="line-height:1.2;{self.text_style}">{getattr(self.provider, 'description', '')}</p>"""
+<p style="line-height:1.2;{self.text_style}">{getattr(self.provider, 'description', '')}</p>"""
         self.provider_summary.value = html_text
 
     def _update_database(self):
         """Update database summary"""
         html_text = f"""<strong style="line-height:1;{self.text_style}">{getattr(self.database, 'name', 'Database')}</strong>
-        <p style="line-height:1.2;{self.text_style}">{getattr(self.database, 'description', '')}</p>"""
+<p style="line-height:1.2;{self.text_style}">{getattr(self.database, 'description', '')}</p>"""
         self.database_summary.value = html_text
 
     def freeze(self):