test: Add more scenarios for the pipeline trigger (#329)

This is to help us catch more bugs, as we missed 1 recently.

CLOSES PLA-151

tests/core/test_browse.py

@@ -8,7 +8,7 @@
 def test_browse_rds_families(data_db):
     setup_with_two_docs(data_db)
     geo = data_db.query(Geography).get(1)
-    expected = 2
+    expected = 1
 
     args = BrowseArgs(
         country_codes=[geo.value],

tests/non_search/app/documents/test_get_document_families.py

@@ -199,7 +199,7 @@ def test_documents_doc_slug_preexisting_objects(
     assert len(family.keys()) == N_FAMILY_OVERVIEW_KEYS
     assert family["title"] == "Fam2"
     assert family["import_id"] == "CCLW.family.2002.0"
-    assert family["geography"] == "South Asia"
+    assert family["geography"] == "AFG"
     assert family["category"] == "Executive"
     assert family["slug"] == "FamSlug2"
     assert family["corpus_id"] == "CCLW.corpus.i00000001.n0000"

tests/non_search/setup_helpers.py

@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 from sqlalchemy.orm import Session
 
 from tests.non_search.dfce_helpers import (
@@ -91,7 +93,7 @@ def get_default_families():
         "title": "Fam2",
         "slug": "FamSlug2",
         "description": "Summary2",
-        "geography_id": 1,
+        "geography_id": 2,
         "category": "Executive",
         "documents": [],
         "metadata": {
@@ -147,6 +149,37 @@ def setup_with_two_docs(db: Session):
     )
 
 
+def setup_with_documents_large_with_families(
+    documents_large: list[Dict[str, Any]],
+    db: Session,
+):
+    # Collection
+    collection1, collection2 = get_default_collections()
+    add_collections(db, collections=[collection1])
+
+    # Family + Document + events
+    family1, family2 = get_default_families()
+
+    family1["metadata"] = {
+        "topic": "Mitigation",
+        "sector": "Economy-wide",
+    }
+
+    split_index = len(documents_large) // 2
+
+    family1["documents"] = documents_large[:split_index]
+    family2["documents"] = documents_large[split_index:]
+    add_families(db, families=[family1, family2])
+
+    # Collection - Family
+    link_collection_family(
+        db,
+        [
+            (collection1["import_id"], family1["import_id"]),
+        ],
+    )
+
+
 def setup_with_two_docs_one_family(db: Session):
     # Collection
     collection1, collection2 = get_default_collections()

tests/unit/app/core/test_pipeline.py

@@ -1,3 +1,5 @@
+from typing import Dict
+
 from sqlalchemy.orm import Session
 
 from app.core.ingestion.pipeline import (
@@ -6,6 +8,7 @@
 )
 from tests.non_search.setup_helpers import (
     setup_docs_with_two_orgs,
+    setup_with_documents_large_with_families,
     setup_with_two_docs_one_family,
     setup_with_two_unpublished_docs,
 )
@@ -44,6 +47,17 @@ def test_generate_pipeline_ingest_input(data_db: Session):
     assert state_rows[0].document_title == "Document2"
 
 
+def test_generate_pipeline_ingest_input_with_fixture(
+    documents_large: list[Dict],
+    data_db: Session,
+):
+    setup_with_documents_large_with_families(documents_large, data_db)
+
+    state_rows = generate_pipeline_ingest_input(data_db)
+
+    assert len(state_rows) == 23
+
+
 def test_generate_pipeline_ingest_input_no_collection_family_link(data_db: Session):
     setup_docs_with_two_orgs(data_db)
 

tests/unit/conftest.py

@@ -0,0 +1,21 @@
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict
+
+import pytest
+
+
+@pytest.fixture
+def documents_large() -> list[Dict[str, Any]]:
+    documents_dir = Path(__file__).parent / "fixtures"
+
+    documents = []
+
+    for file_name in os.listdir(documents_dir):
+        file_path = documents_dir / file_name
+
+        with open(file_path, "r") as f:
+            documents.append(json.load(f))
+
+    return documents

tests/unit/fixtures/CCLW.document.i00000002.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "National Energy and Climate Plan 2019 Draft ",
+  "slug": "national-energy-and-climate-plan-2019-draft_6002",
+  "md5_sum": "653834d6f224328bc3711f5a9d0db0a4",
+  "url": "https://www.energy-community.org/dam/jcr:a0c2b8a8-96c8-4423-993a-537cf51daa65/Draft_NECP_AL_%202021.pdf",
+  "content_type": "application/pdf",
+  "import_id": "CCLW.document.i00000002.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.13.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000006.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Cabinet Decision approving the Plan",
+  "slug": "cabinet-decision-approving-the-plan_3615",
+  "md5_sum": null,
+  "url": "https://www.thepresidency.gov.za/cabinet-statements/statement-cabinet-meeting-19-october-2022",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000006.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.14.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000008.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Cabinet Decision approving the Framework",
+  "slug": "cabinet-decision-approving-the-framework_b837",
+  "md5_sum": null,
+  "url": "https://www.thepresidency.gov.za/cabinet-statements/statement-virtual-cabinet-meeting-31-august-2022",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000008.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.15.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000016.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Decree",
+  "slug": "decree_35ee",
+  "md5_sum": null,
+  "url": "https://www.legifrance.gouv.fr/jorf/id/JORFTEXT000031740341",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000016.n0000",
+  "language_variant": "Original Language",
+  "status": "DELETED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.16.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000018.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Decree",
+  "slug": "decree_dd6c",
+  "md5_sum": "78d5911a24f9c01b1b12671e5d1eb3e0",
+  "url": "https://www.tresor.economie.gouv.fr/Articles/80af1116-2fcd-47d0-ad1d-ea24352e6295/files/273f9026-bbc4-4fc2-ba60-f86f6fe16c1f",
+  "content_type": "application/pdf",
+  "import_id": "CCLW.document.i00000018.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.17.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000026.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "National Adaptation Plan ",
+  "slug": "national-adaptation-plan_4e44",
+  "md5_sum": null,
+  "url": "https://unfccc.int/sites/default/files/resource/NAP-Bosnia-and-Herzegovina%20.pdf",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000026.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.18.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000030.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Law No. 08/L-025 on Protection of Air from Pollution",
+  "slug": "law-no-08-l-025-on-protection-of-air-from-pollution_e1e1",
+  "md5_sum": null,
+  "url": "https://gzk.rks-gov.net/ActDocumentDetail.aspx?ActID=60330",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000030.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.19.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000038.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Growth Acceleration Plan 2022-2026 ",
+  "slug": "growth-acceleration-plan-2022-2026_853e",
+  "md5_sum": null,
+  "url": "https://finance.gov.mk/wp-content/uploads/2022/01/Growth-Acceleration-Plan_final-1.doc",
+  "content_type": null,
+  "import_id": "CCLW.document.i00000038.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.20.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000040.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Decree on the Types of Activities and Gases with a Greenhouse Effect ",
+  "slug": "decree-on-the-types-of-activities-and-gases-with-a-greenhouse-effect_ae49",
+  "md5_sum": null,
+  "url": "https://www.pravno-informacioni-sistem.rs/SlGlasnikPortal/eli/rep/sgrs/vlada/uredba/2022/13/1",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000040.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.21.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000058.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Circular Economy Development Program 2022-2024",
+  "slug": "circular-economy-development-program-2022-2024_46d2",
+  "md5_sum": null,
+  "url": "https://www.pravno-informacioni-sistem.rs/SlGlasnikPortal/eli/rep/sgrs/vlada/drugiakt/2022/137/1",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000058.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.22.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}

tests/unit/fixtures/CCLW.document.i00000066.n0000.json

@@ -0,0 +1,24 @@
+{
+  "title": "Commission Delegated Regulation (EU) 2019/807 of 13 March 2019 supplementing Directive (EU) 2018/2001 of the European Parliament and of the Council as regards the determination of high indirect land-use change-risk feedstock for which a significant expansion of the production area into land with high carbon stock is observed and the certification of low indirect land-use change-risk biofuels, bioliquids and biomass fuels",
+  "slug": "commission-delegated-regulation-eu-2019-807-of-13-march-2019-supplementing-directive-eu-2018-2001-of-the-european-parliament-and-of-the-council-as-regards-the-determination-of-high-indirect-land-use-change-risk-feedstock-for-which-a-significant-expansion-of-the-production-area-into-land-with-high-carbon-stock-is-observed-and-the-certification-of-low-indirect-land-use-change-risk-biofuels-bioliquids-and-biomass-fuels_3404",
+  "md5_sum": null,
+  "url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=uriserv:OJ.L_.2019.133.01.0001.01.ENG&toc=OJ:L:2019:133:TOC|en",
+  "content_type": "text/html",
+  "import_id": "CCLW.document.i00000066.n0000",
+  "language_variant": "Original Language",
+  "status": "PUBLISHED",
+  "metadata": {
+    "role": ["MAIN"],
+    "type": ["Plan"]
+  },
+  "languages": ["eng"],
+  "events": [
+    {
+      "import_id": "CPR.Event.23.0",
+      "title": "Published",
+      "date": "2019-12-25",
+      "type": "Passed/Approved",
+      "status": "OK"
+    }
+  ]
+}