From 46c2d909f6402fdb66faf180516f804476ec3caa Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 4 Nov 2023 21:46:12 -0400 Subject: [PATCH 1/4] Add check for productID in file name --- .gitignore | 1 + backend/handler/igdb_handler.py | 59 +++++++++++++++++++++++--- backend/tasks/update_switch_titledb.py | 19 +++++++++ backend/tasks/utils.py | 6 ++- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 0f06ff0e0..ac9ce7197 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,5 @@ frontend/dev-dist # outside data switch_titledb.json +switch_product_ids.json mame.xml diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py index fa647e983..3741477d8 100644 --- a/backend/handler/igdb_handler.py +++ b/backend/handler/igdb_handler.py @@ -30,11 +30,16 @@ os.path.dirname(__file__), "fixtures", "ps2_opl_index.json" ) -SWITCH_TITLEDB_REGEX: Final = r"^(70[0-9]{12})$" +SWITCH_TITLEDB_REGEX: Final = r"(70[0-9]{12})" SWITCH_TITLEDB_INDEX_FILE: Final = os.path.join( os.path.dirname(__file__), "fixtures", "switch_titledb.json" ) +SWITCH_PRODUCT_ID_REGEX: Final = r"(0100[0-9A-F]{12})" +SWITCH_PRODUCT_ID_FILE: Final = os.path.join( + os.path.dirname(__file__), "fixtures", "switch_product_ids.json" +) + MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml") @@ -72,7 +77,7 @@ def _request(self, url: str, data: str, timeout: int = 120) -> list: if err.response.status_code != 401: log.error(err) return [] # All requests to the IGDB API return a list - + # Attempt to force a token refresh if the token is invalid log.warning("Twitch token invalid: fetching a new one...") token = self.twitch_auth._update_twitch_token() @@ -122,7 +127,11 @@ def _search_cover(self, rom_id: int) -> str: ) cover = pydash.get(covers, "[0]", None) - return DEFAULT_URL_COVER_L if not cover else self._normalize_cover_url(cover["url"]) + return ( + DEFAULT_URL_COVER_L + if not cover + else self._normalize_cover_url(cover["url"]) + ) def _search_screenshots(self, rom_id: int) -> list: screenshots = self._request( @@ -161,7 +170,7 @@ def get_platform(self, p_slug: str): async def get_rom(self, file_name: str, p_igdb_id: int): search_term = get_search_term(file_name) - # Patch support for PS2 OPL flename format + # Support for PS2 OPL flename format match = re.match(PS2_OPL_REGEX, search_term) if p_igdb_id == PS2_IGDB_ID and match: serial_code = match.group(1) @@ -172,8 +181,8 @@ async def get_rom(self, file_name: str, p_igdb_id: int): if index_entry: search_term = index_entry["Name"] # type: ignore - # Patch support for switch titleID filename format - match = re.match(SWITCH_TITLEDB_REGEX, search_term) + # Support for switch titleID filename format + match = re.search(SWITCH_TITLEDB_REGEX, file_name) if p_igdb_id == SWITCH_IGDB_ID and match: title_id = match.group(1) titledb_index = {} @@ -195,8 +204,36 @@ async def get_rom(self, file_name: str, p_igdb_id: int): if index_entry: search_term = index_entry["name"] # type: ignore + # Support for switch productID filename format + match = re.search(SWITCH_PRODUCT_ID_REGEX, file_name) + if p_igdb_id == SWITCH_IGDB_ID and match: + product_id = match.group(1) + product_id_index = {} + + try: + with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json: + product_id_index = json.loads(index_json.read()) + except FileNotFoundError: + log.warning("Fetching the Switch titleDB index file...") + await update_switch_titledb_task.run(force=True) + + try: + with open(SWITCH_PRODUCT_ID_FILE, "r") as index_json: + product_id_index = json.loads(index_json.read()) + except FileNotFoundError: + log.error("Could not fetch the Switch titleDB index file") + finally: + index_entry = product_id_index.get(product_id, None) + if index_entry: + with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json: + titledb_index = json.loads(index_json.read()) + index_entry = titledb_index.get(index_entry, None) + if index_entry: + search_term = index_entry["name"] # type: ignore + + # Support for MAME arcade filename format if p_igdb_id == ARCADE_IGDB_ID: - mame_index = { "menu": { "game": [] } } + mame_index = {"menu": {"game": []}} try: with open(MAME_XML_FILE, "r") as index_xml: @@ -219,6 +256,14 @@ async def get_rom(self, file_name: str, p_igdb_id: int): if index_entry: search_term = index_entry[0].get("description", search_term) + search_term = ( + search_term.replace("\u2122", "") # Remove trademark symbol + .replace("\u00ae", "") # Remove registered symbol + .replace("\u00a9", "") # Remove copywrite symbol + .replace("\u2120", "") # Remove service mark symbol + .strip() # Remove leading and trailing spaces + ) + res = ( self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY) or self._search_rom(uc(search_term), p_igdb_id, EXPANDED_GAME_CATEGORY) diff --git a/backend/tasks/update_switch_titledb.py b/backend/tasks/update_switch_titledb.py index eb48ba1b5..648e667a2 100644 --- a/backend/tasks/update_switch_titledb.py +++ b/backend/tasks/update_switch_titledb.py @@ -1,4 +1,5 @@ import os +import json from pathlib import Path from typing import Final @@ -15,6 +16,13 @@ / "switch_titledb.json" ) +SWITCH_PRODUCT_ID_FILE_PATH: Final = ( + Path(os.path.dirname(__file__)).parent + / "handler" + / "fixtures" + / "switch_product_ids.json" +) + class UpdateSwitchTitleDBTask(RemoteFilePullTask): def __init__(self): @@ -27,5 +35,16 @@ def __init__(self): file_path=FIXTURE_FILE_PATH, ) + async def run(self, force: bool = False): + content = await super().run(force) + if content is None: + return + + index_json = json.loads(content) + product_ids = dict((v['id'],k) for k,v in index_json.items()) + + with open(SWITCH_PRODUCT_ID_FILE_PATH, "wb") as fixture: + fixture.write(json.dumps(product_ids).encode()) + update_switch_titledb_task = UpdateSwitchTitleDBTask() diff --git a/backend/tasks/utils.py b/backend/tasks/utils.py index 140f18793..929bb3ca6 100644 --- a/backend/tasks/utils.py +++ b/backend/tasks/utils.py @@ -83,11 +83,11 @@ def __init__(self, *args, url: str, file_path: str, **kwargs): self.url = url self.file_path = file_path - async def run(self, force: bool = False): + async def run(self, force: bool = False) -> bytes | None: if not self.enabled and not force: log.info(f"Scheduled {self.description} not enabled, unscheduling...") self.unschedule() - return + return None log.info(f"Scheduled {self.description} started...") @@ -99,6 +99,8 @@ async def run(self, force: bool = False): fixture.write(response.content) log.info(f"Scheduled {self.description} done") + return response.content except requests.exceptions.RequestException as e: log.error(f"Scheduled {self.description} failed", exc_info=True) log.error(e) + return None From b99cce243b1d2694f464e3e6abc53813fad4ef06 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 4 Nov 2023 22:04:27 -0400 Subject: [PATCH 2/4] Save the entire value object --- backend/handler/igdb_handler.py | 6 +----- backend/tasks/update_switch_titledb.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py index 3741477d8..0e765f2f5 100644 --- a/backend/handler/igdb_handler.py +++ b/backend/handler/igdb_handler.py @@ -225,11 +225,7 @@ async def get_rom(self, file_name: str, p_igdb_id: int): finally: index_entry = product_id_index.get(product_id, None) if index_entry: - with open(SWITCH_TITLEDB_INDEX_FILE, "r") as index_json: - titledb_index = json.loads(index_json.read()) - index_entry = titledb_index.get(index_entry, None) - if index_entry: - search_term = index_entry["name"] # type: ignore + search_term = index_entry["name"] # type: ignore # Support for MAME arcade filename format if p_igdb_id == ARCADE_IGDB_ID: diff --git a/backend/tasks/update_switch_titledb.py b/backend/tasks/update_switch_titledb.py index 648e667a2..61d4861da 100644 --- a/backend/tasks/update_switch_titledb.py +++ b/backend/tasks/update_switch_titledb.py @@ -41,8 +41,8 @@ async def run(self, force: bool = False): return index_json = json.loads(content) - product_ids = dict((v['id'],k) for k,v in index_json.items()) - + product_ids = dict((v["id"], v) for _k, v in index_json.items()) + with open(SWITCH_PRODUCT_ID_FILE_PATH, "wb") as fixture: fixture.write(json.dumps(product_ids).encode()) From 015e5927d61b7d2bac88d2719a55f48fa130581d Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 4 Nov 2023 23:12:40 -0400 Subject: [PATCH 3/4] Less extensive multi extension regex --- backend/handler/igdb_handler.py | 2 +- backend/utils/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/handler/igdb_handler.py b/backend/handler/igdb_handler.py index 0e765f2f5..97073d00b 100644 --- a/backend/handler/igdb_handler.py +++ b/backend/handler/igdb_handler.py @@ -171,7 +171,7 @@ async def get_rom(self, file_name: str, p_igdb_id: int): search_term = get_search_term(file_name) # Support for PS2 OPL flename format - match = re.match(PS2_OPL_REGEX, search_term) + match = re.match(PS2_OPL_REGEX, file_name) if p_igdb_id == PS2_IGDB_ID and match: serial_code = match.group(1) diff --git a/backend/utils/__init__.py b/backend/utils/__init__.py index 5a84fdb12..7e89df9ab 100644 --- a/backend/utils/__init__.py +++ b/backend/utils/__init__.py @@ -59,7 +59,7 @@ REGIONS_NAME_KEYS = [region[1].lower() for region in REGIONS] TAG_REGEX = r"\(([^)]+)\)|\[([^]]+)\]" -EXTENSION_REGEX = r"\.(\w+(\.\w+)*)$" +EXTENSION_REGEX = r"\.([a-z]+(\.\w+)*)$" def parse_tags(file_name: str) -> tuple: From a3a1eb93231b2383c83456bc56458456782a9891 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 4 Nov 2023 23:30:53 -0400 Subject: [PATCH 4/4] Fix tests --- backend/utils/__init__.py | 2 +- backend/utils/tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/utils/__init__.py b/backend/utils/__init__.py index 7e89df9ab..6a7320efe 100644 --- a/backend/utils/__init__.py +++ b/backend/utils/__init__.py @@ -59,7 +59,7 @@ REGIONS_NAME_KEYS = [region[1].lower() for region in REGIONS] TAG_REGEX = r"\(([^)]+)\)|\[([^]]+)\]" -EXTENSION_REGEX = r"\.([a-z]+(\.\w+)*)$" +EXTENSION_REGEX = r"\.(([a-z]+\.)*\w+)$" def parse_tags(file_name: str) -> tuple: diff --git a/backend/utils/tests/test_utils.py b/backend/utils/tests/test_utils.py index a368767b8..117ade9e2 100644 --- a/backend/utils/tests/test_utils.py +++ b/backend/utils/tests/test_utils.py @@ -58,7 +58,7 @@ def test_get_file_name_with_no_tags(): # This is expected behavior, since the regex is aggressive file_name = "Battle Stadium D.O.N.zip" - assert gfnwt(file_name) == "Battle Stadium D" + assert gfnwt(file_name) == "Battle Stadium D.O.N" def test_get_file_extension():