Fix broken pipeline due to CardTable2 "status" overhaul

.github/workflows/merge.yaml

@@ -61,6 +61,7 @@ jobs:
             --assignments ../../src/assignments/assignments.yaml \
             --tcg ../../../tcg.vector.json \
             --ocg ../../../ocg.vector.json \
+            --unreleased ../../../yaml-yugipedia/semantic-mediawiki/unreleased.csv \
             --ko-official ../../../yaml-yugi-ko/_site/ocg.csv \
             --ko-override ../../../yaml-yugi-ko/ocg-override.csv \
             --ko-prerelease ../../../yaml-yugi-ko/ocg-prerelease.csv \

src/common.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: © 2022–2023 Kevin Lu
+# SPDX-FileCopyrightText: © 2022–2024 Kevin Lu
 # SPDX-Licence-Identifier: AGPL-3.0-or-later
 from csv import DictReader
 import json
@@ -324,3 +324,11 @@ def replace_interlinear_annotations(name: str) -> str:
         .replace("\ufffa", "<rt>")
         .replace("\ufffb", "</rt></ruby>")
     )
+
+
+def load_unreleased_csv(filename: Optional[str]) -> Dict[str, Dict[str, str]]:
+    if not filename:
+        return {}
+    with open(filename) as f:
+        reader = DictReader(f)
+        return {row["English name"]: row for row in reader}

src/job_ocgtcg.py

@@ -22,6 +22,7 @@
     transform_texts,
     write,
     load_ko_csv,
+    load_unreleased_csv,
 )
 
 module_logger = logging.getLogger(__name__)
@@ -63,6 +64,9 @@ def transform_structure(
         "limitation_text" in wikitext
         or
         # Boss Duel cards
+        wikitext.get("jp_sets", "").startswith("BD-JP")
+        or
+        # Deprecated: https://yugipedia.com/wiki/Category:Cards_with_a_manual_status
         wikitext.get("ocg_status") == "Illegal"
         or
         # Details unavailable for a new leak
@@ -113,9 +117,24 @@ def transform_structure(
 
 def annotate_limit_regulation(
     document: Dict[str, Any],
+    unreleased: Dict[str, Dict[str, str]],
     tcg_vector: Optional[Dict[str, int]],
     ocg_vector: Optional[Dict[str, int]],
 ) -> None:
+    if (
+        (release := unreleased.get(document["name"]["en"]))
+        # Exclude The Seal of Orichalcos (UDE promo) and some others
+        and release.get("OCG status") != "Illegal"
+        and release.get("TCG status") != "Illegal"
+    ):
+        document["limit_regulation"]["tcg"] = release.get(
+            "TCG status", "Not yet released"
+        )
+        document["limit_regulation"]["ocg"] = release.get(
+            "OCG status", "Not yet released"
+        )
+        if speed := release.get("TCG Speed Duel status"):
+            document["limit_regulation"]["speed"] = speed
     if (
         tcg_vector
         and document["konami_id"]
@@ -393,6 +412,7 @@ def job(
     assignment_file: Optional[str] = None,
     tcg_vector: Optional[Dict[str, int]] = None,
     ocg_vector: Optional[Dict[str, int]] = None,
+    unreleased_csv: Optional[str] = None,
     ko_official_csv: Optional[str] = None,
     ko_override_csv: Optional[str] = None,
     ko_prerelease_csv: Optional[str] = None,
@@ -402,6 +422,7 @@ def job(
     yaml = YAML()
     yaml.width = sys.maxsize
     assignments = load_assignments(yaml, assignment_file) if assignment_file else None
+    unreleased = load_unreleased_csv(unreleased_csv)
     ko_official = load_ko_csv("konami_id", ko_official_csv)
     ko_override = load_ko_csv("konami_id", ko_override_csv)
     ko_prerelease = load_ko_csv("yugipedia_page_id", ko_prerelease_csv)  # noqa: F841
@@ -427,7 +448,7 @@ def job(
         properties["yugipedia_page_id"] = page_id
         document = transform_structure(logger, properties)
         if document:
-            annotate_limit_regulation(document, tcg_vector, ocg_vector)
+            annotate_limit_regulation(document, unreleased, tcg_vector, ocg_vector)
             if ko_official:
                 replace_with_official(logger, document, ko_official, "ko")
             if master_duel:

src/main_ocgtcg.py

@@ -17,6 +17,9 @@
 parser.add_argument(
     "--ocg", help="OCG Forbidden & Limited List, English name vector JSON"
 )
+parser.add_argument(
+    "--unreleased", help="Semantic MediaWiki unreleased cards CSV export"
+)
 parser.add_argument("--ko-official", help="yaml-yugi-ko official database CSV")
 parser.add_argument("--ko-override", help="yaml-yugi-ko ocg-override.csv")
 parser.add_argument("--ko-prerelease", help="yaml-yugi-ko ocg-prerelease.csv")
@@ -60,6 +63,7 @@ def main() -> None:
         args.assignments,
         tcg,
         ocg,
+        args.unreleased,
         args.ko_official,
         args.ko_override,
         args.ko_prerelease,