From e74982cadcd23e2c6a7cec3e8fee74b6491d7ca2 Mon Sep 17 00:00:00 2001 From: Zdenek Styblik Date: Sun, 30 Jun 2024 21:51:55 +0200 Subject: [PATCH] WIP: Move CachedData and HTTPSource into separate files TODO: * CI * split-off tests * cleanup everything and every reference to rss2irc --- cache_stats.py | 5 +- gh2slack.py | 14 +-- git_commits2slack.py | 6 +- lib/__init__.py | 4 + lib/cached_data.py | 45 ++++++++++ lib/config_options.py | 9 ++ lib/http_source.py | 41 +++++++++ migrations/convert_cache_to_dataclass_v1.py | 11 +-- migrations/convert_cache_to_dataclass_v2.py | 14 +-- .../test_convert_cache_to_dataclass_v1.py | 3 +- phpbb2slack.py | 27 +++--- rss2irc.py | 86 +++---------------- rss2slack.py | 11 +-- tests/test_cache_stats.py | 3 +- tests/test_gh2slack.py | 5 +- tests/test_phpbb2slack.py | 14 +-- tests/test_rss2irc.py | 51 +++++------ tests/test_rss2slack.py | 10 ++- 18 files changed, 206 insertions(+), 153 deletions(-) create mode 100644 lib/__init__.py create mode 100644 lib/cached_data.py create mode 100644 lib/config_options.py create mode 100644 lib/http_source.py diff --git a/cache_stats.py b/cache_stats.py index 2772c27..a835313 100755 --- a/cache_stats.py +++ b/cache_stats.py @@ -10,6 +10,7 @@ from dataclasses import dataclass import rss2irc +from lib import CachedData BUCKET_COUNT = 10 @@ -24,7 +25,7 @@ class Bucket: def calc_distribution( - logger: logging.Logger, cache: rss2irc.CachedData, buckets + logger: logging.Logger, cache: CachedData, buckets ) -> int: """Calculate item distribution inside cache.""" keys = list(buckets.keys()) @@ -70,7 +71,7 @@ def get_timestamp(data) -> int: def get_timestamp_minmax( - logger: logging.Logger, cache: rss2irc.CachedData + logger: logging.Logger, cache: CachedData ) -> (int, int, int): """Return timestamp min, max and no. of errors.""" ts_min = 99999999999 diff --git a/gh2slack.py b/gh2slack.py index 8037c23..3a9989c 100755 --- a/gh2slack.py +++ b/gh2slack.py @@ -18,6 +18,8 @@ import rss2irc # noqa: I202 import rss2slack +from lib import CachedData +from lib import config_options ALIASES = { "issues": "issue", @@ -101,7 +103,7 @@ def gh_parse_next_page(link_header: str) -> str: def gh_request( - logger: logging.Logger, url: str, timeout: int = rss2irc.HTTP_TIMEOUT + logger: logging.Logger, url: str, timeout: int = config_options.HTTP_TIMEOUT ) -> List: """Return list of responses from GitHub. @@ -223,7 +225,7 @@ def parse_args() -> argparse.Namespace: "--cache-expiration", dest="cache_expiration", type=int, - default=rss2irc.CACHE_EXPIRATION, + default=config_options.CACHE_EXPIRATION, help="Time, in seconds, for how long to keep items " "in cache.", ) parser.add_argument( @@ -275,9 +277,9 @@ def parse_args() -> argparse.Namespace: "--slack-timeout", dest="slack_timeout", type=int, - default=rss2irc.HTTP_TIMEOUT, + default=config_options.HTTP_TIMEOUT, help="Slack API Timeout. Defaults to {:d} seconds.".format( - rss2irc.HTTP_TIMEOUT + config_options.HTTP_TIMEOUT ), ) parser.add_argument( @@ -303,7 +305,7 @@ def parse_args() -> argparse.Namespace: def process_page_items( logger: logging.Logger, - cache: rss2irc.CachedData, + cache: CachedData, pages: List, expiration: int, repository_url: str, @@ -347,7 +349,7 @@ def process_page_items( return to_publish -def scrub_items(logger: logging.Logger, cache: rss2irc.CachedData) -> None: +def scrub_items(logger: logging.Logger, cache: CachedData) -> None: """Scrub cache and remove expired items.""" time_now = int(time.time()) for key in list(cache.items.keys()): diff --git a/git_commits2slack.py b/git_commits2slack.py index b3196fe..1fad68c 100755 --- a/git_commits2slack.py +++ b/git_commits2slack.py @@ -16,8 +16,8 @@ from typing import Dict from typing import List -import rss2irc import rss2slack +from lib import config_options RE_GIT_AUTD = re.compile(r"^Already up-to-date.$") RE_GIT_UPDATING = re.compile(r"^Updating [a-z0-9]+", re.I) @@ -254,9 +254,9 @@ def parse_args() -> argparse.Namespace: "--slack-timeout", dest="slack_timeout", type=int, - default=rss2irc.HTTP_TIMEOUT, + default=config_options.HTTP_TIMEOUT, help="Slack API Timeout. Defaults to {:d} seconds.".format( - rss2irc.HTTP_TIMEOUT + config_options.HTTP_TIMEOUT ), ) parser.add_argument( diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..605d21b --- /dev/null +++ b/lib/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +"""FIXME.""" +from .cached_data import CachedData # noqa: F401 +from .http_source import HTTPSource # noqa: F401 diff --git a/lib/cached_data.py b/lib/cached_data.py new file mode 100644 index 0000000..d8e324e --- /dev/null +++ b/lib/cached_data.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Code related to Cache. + +I love how black and reorder-python-imports play nicely together and no +workarounds are needed. +""" +import time +from dataclasses import dataclass +from dataclasses import field + +from .config_options import DATA_SOURCE_EXPIRATION +from .http_source import HTTPSource + + +@dataclass +class CachedData: + """CachedData represents locally cached data and state.""" + + data_sources: dict = field(default_factory=dict) + items: dict = field(default_factory=dict) + + def get_source_by_url(self, url: str) -> HTTPSource: + """Return source by URL. + + If source doesn't exist, it will be created. + """ + source = self.data_sources.get(url, None) + if source: + source.last_used_ts = int(time.time()) + return source + + self.data_sources[url] = HTTPSource( + last_used_ts=int(time.time()), url=url + ) + return self.get_source_by_url(url) + + def scrub_data_sources( + self, expiration: int = DATA_SOURCE_EXPIRATION + ) -> None: + """Delete expired data sources.""" + now = int(time.time()) + for key in list(self.data_sources.keys()): + diff = now - self.data_sources[key].last_used_ts + if int(diff) > expiration: + self.data_sources.pop(key) diff --git a/lib/config_options.py b/lib/config_options.py new file mode 100644 index 0000000..ab2edab --- /dev/null +++ b/lib/config_options.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Common configuration options. + +I love how black and reorder-python-imports play nicely together and no +workarounds are needed. +""" +CACHE_EXPIRATION = 86400 # seconds +DATA_SOURCE_EXPIRATION = 30 * 86400 # seconds +HTTP_TIMEOUT = 30 # seconds diff --git a/lib/http_source.py b/lib/http_source.py new file mode 100644 index 0000000..014d9b0 --- /dev/null +++ b/lib/http_source.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Code related to HTTP Source. + +I love how black and reorder-python-imports play nicely together and no +workarounds are needed. +""" +from dataclasses import dataclass +from dataclasses import field +from typing import Dict + + +@dataclass +class HTTPSource: + """Class represents HTTP data source.""" + + http_etag: str = field(default_factory=str) + http_last_modified: str = field(default_factory=str) + last_used_ts: int = 0 + url: str = field(default_factory=str) + + def extract_caching_headers(self, headers: Dict[str, str]) -> None: + """Extract cache related headers from given dict.""" + self.http_etag = "" + self.http_last_modified = "" + for key, value in headers.items(): + key = key.lower() + if key == "etag": + self.http_etag = value + elif key == "last-modified": + self.http_last_modified = value + + def make_caching_headers(self) -> Dict[str, str]: + """Return cache related headers as a dict.""" + headers = {} + if self.http_etag: + headers["if-none-match"] = self.http_etag + + if self.http_last_modified: + headers["if-modified-since"] = self.http_last_modified + + return headers diff --git a/migrations/convert_cache_to_dataclass_v1.py b/migrations/convert_cache_to_dataclass_v1.py index 33baa1f..d51bbe8 100755 --- a/migrations/convert_cache_to_dataclass_v1.py +++ b/migrations/convert_cache_to_dataclass_v1.py @@ -15,13 +15,14 @@ import sys from importlib.machinery import SourceFileLoader -# NOTICE: An ugly hack in order to be able to import CachedData class from -# rss2irc. I'm real sorry about this, son. +# NOTICE: An ugly hack in order to be able to import CachedData class. +# I'm real sorry about this, son. # NOTE: Sadly, importlib.util and spec didn't cut it. Also, I'm out of time on # this. Therefore, see you again in the future once this ceases to work. SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) -rss2irc_module_path = os.path.join(SCRIPT_PATH, "..", "rss2irc.py") -rss2irc = SourceFileLoader("rss2irc", rss2irc_module_path).load_module() +lib_module_path = os.path.join(SCRIPT_PATH, "..", "lib", "__init__.py") +lib = SourceFileLoader("lib", lib_module_path).load_module() +CachedData = lib.cached_data.CachedData def main(): @@ -50,7 +51,7 @@ def main(): logger.info("Create backup file '%s' from '%s'.", bak_file, args.cache) shutil.copy2(args.cache, bak_file) - new_cache = rss2irc.CachedData() + new_cache = CachedData() for key, value in cache.items(): new_cache.items[key] = value diff --git a/migrations/convert_cache_to_dataclass_v2.py b/migrations/convert_cache_to_dataclass_v2.py index 3c70887..6de9594 100644 --- a/migrations/convert_cache_to_dataclass_v2.py +++ b/migrations/convert_cache_to_dataclass_v2.py @@ -14,14 +14,16 @@ import sys from importlib.machinery import SourceFileLoader -# NOTICE: An ugly hack in order to be able to import CachedData class from -# rss2irc. I'm real sorry about this, son. -# NOTE: Sadly, importlib.util and spec didn't cut it. As usual, I'm out of time -# on this. Therefore, see you again in the future once this ceases to work. +# NOTICE: An ugly hack in order to be able to import CachedData class. +# I'm real sorry about this, son. +# NOTE: Sadly, importlib.util and spec didn't cut it. Also, I'm out of time on +# this. Therefore, see you again in the future once this ceases to work. SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) +lib_module_path = os.path.join(SCRIPT_PATH, "..", "lib", "__init__.py") +lib = SourceFileLoader("lib", lib_module_path).load_module() rss2irc_module_path = os.path.join(SCRIPT_PATH, "..", "rss2irc.py") rss2irc = SourceFileLoader("rss2irc", rss2irc_module_path).load_module() -CachedData = rss2irc.CachedData +CachedData = lib.cached_data.CachedData def main(): @@ -43,7 +45,7 @@ def main(): logger.info("Create backup file '%s' from '%s'.", bak_file, args.cache) shutil.copy2(args.cache, bak_file) - new_cache = rss2irc.CachedData() + new_cache = CachedData() for key, value in cache.items.items(): new_cache.items[key] = value diff --git a/migrations/tests/test_convert_cache_to_dataclass_v1.py b/migrations/tests/test_convert_cache_to_dataclass_v1.py index 53546bb..f01667c 100644 --- a/migrations/tests/test_convert_cache_to_dataclass_v1.py +++ b/migrations/tests/test_convert_cache_to_dataclass_v1.py @@ -9,6 +9,7 @@ import pytest import rss2irc # noqa:I202 +from lib import CachedData SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -57,7 +58,7 @@ def test_migration(fixture_cache_file, fixture_bak_cleanup): with open(fixture_cache_file, "wb") as fhandle: pickle.dump(test_data, fhandle, pickle.HIGHEST_PROTOCOL) - expected_cache = rss2irc.CachedData( + expected_cache = CachedData( items={ "test1": 1234, "test2": 0, diff --git a/phpbb2slack.py b/phpbb2slack.py index 9452e20..07f26ae 100755 --- a/phpbb2slack.py +++ b/phpbb2slack.py @@ -14,10 +14,9 @@ import feedparser import rss2irc # noqa: I202 -import rss2slack - -CACHE_EXPIRATION = 86400 # seconds -HTTP_TIMEOUT = 30 # seconds +import rss2slack # noqa: I202 +from lib import CachedData # noqa: I202 +from lib import config_options # noqa: I202 def format_message( @@ -163,7 +162,7 @@ def parse_args() -> argparse.Namespace: "--cache-expiration", dest="cache_expiration", type=int, - default=CACHE_EXPIRATION, + default=config_options.CACHE_EXPIRATION, help="Time, in seconds, for how long to keep items in cache.", ) parser.add_argument( @@ -194,8 +193,10 @@ def parse_args() -> argparse.Namespace: "--rss-http-timeout", dest="rss_http_timeout", type=int, - default=HTTP_TIMEOUT, - help="HTTP Timeout. Defaults to {:d} seconds.".format(HTTP_TIMEOUT), + default=config_options.HTTP_TIMEOUT, + help="HTTP Timeout. Defaults to {:d} seconds.".format( + config_options.HTTP_TIMEOUT + ), ) parser.add_argument( "--slack-base-url", @@ -215,9 +216,9 @@ def parse_args() -> argparse.Namespace: "--slack-timeout", dest="slack_timeout", type=int, - default=HTTP_TIMEOUT, + default=config_options.HTTP_TIMEOUT, help="Slack API Timeout. Defaults to {:d} seconds.".format( - HTTP_TIMEOUT + config_options.HTTP_TIMEOUT ), ) parser.add_argument( @@ -274,9 +275,9 @@ def parse_news(data: str, authors: List[str]) -> Dict: def prune_news( logger: logging.Logger, - cache: rss2irc.CachedData, + cache: CachedData, news: Dict[str, Dict], - expiration: int = CACHE_EXPIRATION, + expiration: int = config_options.CACHE_EXPIRATION, ) -> None: """Prune news which already are in cache.""" item_expiration = int(time.time()) + expiration @@ -292,7 +293,7 @@ def prune_news( news.pop(key) -def scrub_items(logger: logging.Logger, cache: rss2irc.CachedData) -> None: +def scrub_items(logger: logging.Logger, cache: CachedData) -> None: """Scrub cache and remove expired items.""" time_now = int(time.time()) for key in list(cache.items.keys()): @@ -312,7 +313,7 @@ def scrub_items(logger: logging.Logger, cache: rss2irc.CachedData) -> None: def update_items_expiration( - cache: rss2irc.CachedData, news: Dict, expiration: int + cache: CachedData, news: Dict, expiration: int ) -> None: """Update cache contents.""" item_expiration = int(time.time()) + expiration diff --git a/rss2irc.py b/rss2irc.py index 5471ca4..bcd1dcf 100755 --- a/rss2irc.py +++ b/rss2irc.py @@ -12,8 +12,6 @@ import sys import time import traceback -from dataclasses import dataclass -from dataclasses import field from typing import BinaryIO from typing import Dict from typing import Tuple @@ -21,74 +19,8 @@ import feedparser import requests -CACHE_EXPIRATION = 86400 # seconds -DATA_SOURCE_EXPIRATION = 30 * 86400 # seconds -HTTP_TIMEOUT = 30 # seconds - - -@dataclass -class HTTPSource: - """Class represents HTTP data source.""" - - http_etag: str = field(default_factory=str) - http_last_modified: str = field(default_factory=str) - last_used_ts: int = 0 - url: str = field(default_factory=str) - - def extract_caching_headers(self, headers: Dict[str, str]) -> None: - """Extract cache related headers from given dict.""" - self.http_etag = "" - self.http_last_modified = "" - for key, value in headers.items(): - key = key.lower() - if key == "etag": - self.http_etag = value - elif key == "last-modified": - self.http_last_modified = value - - def make_caching_headers(self) -> Dict[str, str]: - """Return cache related headers as a dict.""" - headers = {} - if self.http_etag: - headers["if-none-match"] = self.http_etag - - if self.http_last_modified: - headers["if-modified-since"] = self.http_last_modified - - return headers - - -@dataclass -class CachedData: - """CachedData represents locally cached data and state.""" - - data_sources: dict = field(default_factory=dict) - items: dict = field(default_factory=dict) - - def get_source_by_url(self, url: str) -> HTTPSource: - """Return source by URL. - - If source doesn't exist, it will be created. - """ - source = self.data_sources.get(url, None) - if source: - source.last_used_ts = int(time.time()) - return source - - self.data_sources[url] = HTTPSource( - last_used_ts=int(time.time()), url=url - ) - return self.get_source_by_url(url) - - def scrub_data_sources( - self, expiration: int = DATA_SOURCE_EXPIRATION - ) -> None: - """Delete expired data sources.""" - now = int(time.time()) - for key in list(self.data_sources.keys()): - diff = now - self.data_sources[key].last_used_ts - if int(diff) > expiration: - self.data_sources.pop(key) +from lib import CachedData # noqa:I202 +from lib import config_options def format_message( @@ -114,7 +46,7 @@ def format_message( def get_rss( logger: logging.Logger, url: str, - timeout: int = HTTP_TIMEOUT, + timeout: int = config_options.HTTP_TIMEOUT, extra_headers: Dict = None, ) -> requests.models.Response: """Return body of given URL as a string.""" @@ -215,8 +147,10 @@ def parse_args() -> argparse.Namespace: "--rss-http-timeout", dest="rss_http_timeout", type=int, - default=HTTP_TIMEOUT, - help="HTTP Timeout. Defaults to {:d} seconds.".format(HTTP_TIMEOUT), + default=config_options.HTTP_TIMEOUT, + help="HTTP Timeout. Defaults to {:d} seconds.".format( + config_options.HTTP_TIMEOUT + ), ) parser.add_argument( "--handle", @@ -243,7 +177,7 @@ def parse_args() -> argparse.Namespace: "--cache-expiration", dest="cache_expiration", type=int, - default=CACHE_EXPIRATION, + default=config_options.CACHE_EXPIRATION, help="Time, in seconds, for how long to keep items in cache.", ) parser.add_argument( @@ -287,7 +221,7 @@ def prune_news( logger: logging.Logger, cache: CachedData, news: Dict[str, Tuple[str, str]], - expiration: int = CACHE_EXPIRATION, + expiration: int = config_options.CACHE_EXPIRATION, ) -> None: """Prune news which already are in cache.""" item_expiration = int(time.time()) + expiration @@ -349,7 +283,7 @@ def scrub_items(logger: logging.Logger, cache: CachedData) -> None: def update_items_expiration( cache: CachedData, news: Dict[str, Tuple[str, str]], - expiration: int = CACHE_EXPIRATION, + expiration: int = config_options.CACHE_EXPIRATION, ) -> None: """Update expiration of items in cache based on news dict.""" item_expiration = int(time.time()) + expiration diff --git a/rss2slack.py b/rss2slack.py index 4bbebdd..90e0232 100755 --- a/rss2slack.py +++ b/rss2slack.py @@ -16,6 +16,7 @@ from slack import WebClient import rss2irc # noqa: I100, I202 +from lib import config_options SLACK_BASE_URL = WebClient.BASE_URL @@ -152,7 +153,7 @@ def parse_args() -> argparse.Namespace: "--cache-expiration", dest="cache_expiration", type=int, - default=rss2irc.CACHE_EXPIRATION, + default=config_options.CACHE_EXPIRATION, help="Time, in seconds, for how long to keep items in cache.", ) parser.add_argument( @@ -183,9 +184,9 @@ def parse_args() -> argparse.Namespace: "--rss-http-timeout", dest="rss_http_timeout", type=int, - default=rss2irc.HTTP_TIMEOUT, + default=config_options.HTTP_TIMEOUT, help="HTTP Timeout. Defaults to {:d} seconds.".format( - rss2irc.HTTP_TIMEOUT + config_options.HTTP_TIMEOUT ), ) parser.add_argument( @@ -206,9 +207,9 @@ def parse_args() -> argparse.Namespace: "--slack-timeout", dest="slack_timeout", type=int, - default=rss2irc.HTTP_TIMEOUT, + default=config_options.HTTP_TIMEOUT, help="Slack API Timeout. Defaults to {:d} seconds.".format( - rss2irc.HTTP_TIMEOUT + config_options.HTTP_TIMEOUT ), ) parser.add_argument( diff --git a/tests/test_cache_stats.py b/tests/test_cache_stats.py index 53874a1..f1b8e88 100644 --- a/tests/test_cache_stats.py +++ b/tests/test_cache_stats.py @@ -8,6 +8,7 @@ import cache_stats # noqa:I202 import rss2irc # noqa:I202 +from lib import CachedData # noqa:I202 SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -16,7 +17,7 @@ def test_main_ideal(fixture_cache_file): """Simple run-through test.""" rss_url = "https://example.com/rss" - cache = rss2irc.CachedData() + cache = CachedData() cache.items = { "a": int(time.time()), "b": int(time.time()), diff --git a/tests/test_gh2slack.py b/tests/test_gh2slack.py index ce27125..815f531 100644 --- a/tests/test_gh2slack.py +++ b/tests/test_gh2slack.py @@ -14,6 +14,7 @@ import gh2slack # noqa:I100,I202 import rss2irc # noqa:I100,I202 +from lib import CachedData # noqa:I100,I202 SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -351,7 +352,7 @@ def test_process_page_items(): ], ] repository_url = "http://example.com" - cache = rss2irc.CachedData( + cache = CachedData( items={ "http://example.com/bar": { "expiration": 0, @@ -391,7 +392,7 @@ def test_process_page_items(): def test_scrub_items(): """Test scrub_items().""" item_expiration = int(time.time()) + 60 - test_cache = rss2irc.CachedData( + test_cache = CachedData( items={ "foo": { "expiration": item_expiration, diff --git a/tests/test_phpbb2slack.py b/tests/test_phpbb2slack.py index b3e2b73..f46222f 100644 --- a/tests/test_phpbb2slack.py +++ b/tests/test_phpbb2slack.py @@ -12,6 +12,8 @@ import phpbb2slack # noqa:I100,I202 import rss2irc +from lib import CachedData +from lib import config_options ITEM_EXPIRATION = int(time.time()) SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -156,13 +158,15 @@ def test_main_ideal( ) fixture_http_server.capture_requests = True - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(rss_url) source1.http_etag = "" source1.http_last_modified = "" source1.last_used_ts = int(time.time()) - 2 * 86400 source2 = cache.get_source_by_url("http://delete.example.com") - source2.last_used_ts = int(time.time()) - 2 * rss2irc.DATA_SOURCE_EXPIRATION + source2.last_used_ts = ( + int(time.time()) - 2 * config_options.DATA_SOURCE_EXPIRATION + ) rss2irc.write_cache(cache, fixture_cache_file) # authors_file = os.path.join(SCRIPT_PATH, "files", "authors.txt") @@ -261,7 +265,7 @@ def test_main_cache_hit( ) fixture_http_server.capture_requests = True - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(rss_url) source1.http_etag = "pytest_etag" source1.http_last_modified = "pytest_lm" @@ -358,7 +362,7 @@ def test_parse_news(): "cache,expected_cache", [ ( - rss2irc.CachedData( + CachedData( items={ "foo": { "expiration": get_item_expiration() + 60, @@ -399,7 +403,7 @@ def test_scrub_items(cache, expected_cache): "comments_cnt": 20, }, }, - rss2irc.CachedData( + CachedData( items={ "http://example.com": { "expiration": 0, diff --git a/tests/test_rss2irc.py b/tests/test_rss2irc.py index 98d772a..0103f99 100644 --- a/tests/test_rss2irc.py +++ b/tests/test_rss2irc.py @@ -10,6 +10,9 @@ import pytest import rss2irc # noqa:I202 +from lib import CachedData # noqa:I202 +from lib import config_options # noqa:I202 +from lib import HTTPSource # noqa:I202 SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -19,23 +22,19 @@ [ # No attrs should bet set ( - rss2irc.HTTPSource(), + HTTPSource(), {}, {"etag": "", "last_modified": ""}, ), # Reset aatrs ( - rss2irc.HTTPSource( - http_etag="et_test", http_last_modified="lm_test" - ), + HTTPSource(http_etag="et_test", http_last_modified="lm_test"), {"header1": "firt", "header2": "second"}, {"etag": "", "last_modified": ""}, ), # Set attrs ( - rss2irc.HTTPSource( - http_etag="et_test", http_last_modified="lm_test" - ), + HTTPSource(http_etag="et_test", http_last_modified="lm_test"), {"ETag": "test123", "Last-Modified": "abc123", "some": "header"}, {"etag": "test123", "last_modified": "abc123"}, ), @@ -52,21 +51,19 @@ def test_http_source_extract_caching_headers(source, input_data, expected): "source,expected", [ ( - rss2irc.HTTPSource(), + HTTPSource(), {}, ), ( - rss2irc.HTTPSource(http_etag="et_test"), + HTTPSource(http_etag="et_test"), {"if-none-match": "et_test"}, ), ( - rss2irc.HTTPSource(http_last_modified="lm_test"), + HTTPSource(http_last_modified="lm_test"), {"if-modified-since": "lm_test"}, ), ( - rss2irc.HTTPSource( - http_etag="et_test", http_last_modified="lm_test" - ), + HTTPSource(http_etag="et_test", http_last_modified="lm_test"), {"if-modified-since": "lm_test", "if-none-match": "et_test"}, ), ], @@ -82,11 +79,11 @@ def test_cache_get_source_by_url(mock_time): """Test that CachedData.get_source_by_url() sets last_used_ts attr.""" mock_time.return_value = 1717428213 url = "http://example.com" - source = rss2irc.HTTPSource( + source = HTTPSource( last_used_ts=0, url=url, ) - cache = rss2irc.CachedData( + cache = CachedData( data_sources={ url: source, } @@ -98,7 +95,7 @@ def test_cache_get_source_by_url(mock_time): def test_cache_scrub_data_sources_empty(cache): """Test that CachedData.scrub_data_sources() when there are no sources.""" - cache = rss2irc.CachedData() + cache = CachedData() assert not cache.data_sources cache.scrub_data_sources() assert not cache.data_sources @@ -108,10 +105,12 @@ def test_cache_scrub_data_sources(cache): """Test that CachedData.scrub_data_sources() expired source is removed.""" source1_url = "http://ww1.example.com" source2_url = "http://ww2.example.com" - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(source1_url) assert source1.last_used_ts > 0 - source1.last_used_ts = int(time.time()) - 2 * rss2irc.DATA_SOURCE_EXPIRATION + source1.last_used_ts = ( + int(time.time()) - 2 * config_options.DATA_SOURCE_EXPIRATION + ) source2 = cache.get_source_by_url(source2_url) assert source2.last_used_ts > 0 @@ -183,13 +182,15 @@ def test_main_ideal( {"ETag": "pytest_etag", "Last-Modified": "pytest_lm"}, ) - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(rss_url) source1.http_etag = "" source1.http_last_modified = "" source1.last_used_ts = int(time.time()) - 2 * 86400 source2 = cache.get_source_by_url("http://delete.example.com") - source2.last_used_ts = int(time.time()) - 2 * rss2irc.DATA_SOURCE_EXPIRATION + source2.last_used_ts = ( + int(time.time()) - 2 * config_options.DATA_SOURCE_EXPIRATION + ) rss2irc.write_cache(cache, fixture_cache_file) logger = logging.getLogger("test") @@ -283,7 +284,7 @@ def test_main_cache_operations( {"ETag": "pytest_etag", "Last-Modified": "pytest_lm"}, ) - cache = rss2irc.CachedData() + cache = CachedData() cache.items[cache_key] = frozen_ts + 60 cache.items["https://expired.example.com"] = 123456 source1 = cache.get_source_by_url(rss_url) @@ -291,7 +292,7 @@ def test_main_cache_operations( source1.http_last_modified = "" source1.last_used_ts = frozen_ts - 2 * 86400 source2 = cache.get_source_by_url("http://delete.example.com") - source2.last_used_ts = frozen_ts - 2 * rss2irc.DATA_SOURCE_EXPIRATION + source2.last_used_ts = frozen_ts - 2 * config_options.DATA_SOURCE_EXPIRATION rss2irc.write_cache(cache, fixture_cache_file) logger = logging.getLogger("test") @@ -340,7 +341,7 @@ def test_main_cache_operations( print("Cache: {}".format(cache)) assert list(cache.items.keys()) == expected_cache_keys # Verify item expiration is updated - assert cache.items[cache_key] == frozen_ts + rss2irc.CACHE_EXPIRATION + assert cache.items[cache_key] == frozen_ts + config_options.CACHE_EXPIRATION # Verify data sources assert rss_url in cache.data_sources.keys() source = cache.get_source_by_url(rss_url) @@ -378,7 +379,7 @@ def test_main_cache_hit( }, ) - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(rss_url) source1.http_etag = "pytest_etag" source1.http_last_modified = "pytest_last_modified" @@ -471,7 +472,7 @@ def test_scrub_items(): logger.disabled = True item_expiration = int(time.time()) + 60 - test_cache = rss2irc.CachedData( + test_cache = CachedData( items={ "foo": item_expiration, "bar": int(time.time()) - 3600, diff --git a/tests/test_rss2slack.py b/tests/test_rss2slack.py index da3f146..be91108 100644 --- a/tests/test_rss2slack.py +++ b/tests/test_rss2slack.py @@ -12,6 +12,8 @@ import rss2irc # noqa:I100,I202 import rss2slack # noqa:I100,I202 +from lib import CachedData +from lib import config_options SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) @@ -150,13 +152,15 @@ def test_main_ideal( ) fixture_http_server.capture_requests = True - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(rss_url) source1.http_etag = "" source1.http_last_modified = "" source1.last_used_ts = int(time.time()) - 2 * 86400 source2 = cache.get_source_by_url("http://delete.example.com") - source2.last_used_ts = int(time.time()) - 2 * rss2irc.DATA_SOURCE_EXPIRATION + source2.last_used_ts = ( + int(time.time()) - 2 * config_options.DATA_SOURCE_EXPIRATION + ) rss2irc.write_cache(cache, fixture_cache_file) # @@ -260,7 +264,7 @@ def test_main_cache_hit( ) fixture_http_server.capture_requests = True - cache = rss2irc.CachedData() + cache = CachedData() source1 = cache.get_source_by_url(rss_url) source1.http_etag = "pytest_etag" source1.http_last_modified = "pytest_lm"