diff --git a/iaso/admin.py b/iaso/admin.py index 1239b2e049..8e0ebdeab6 100644 --- a/iaso/admin.py +++ b/iaso/admin.py @@ -27,6 +27,7 @@ AlgorithmRun, BulkCreateUserCsvFile, DataSource, + DataSourceVersionsSynchronization, Device, DeviceOwnership, DevicePosition, @@ -508,6 +509,7 @@ class TaskAdmin(admin.ModelAdmin): list_filter = ("account", "status", "name") readonly_fields = ("stacktrace", "created_at", "result") formfield_overrides = {models.JSONField: {"widget": IasoJSONEditorWidget}} + search_fields = ("name",) def result_message(self, task): return task.result and task.result.get("message", "") @@ -526,7 +528,7 @@ def get_queryset(self, request): @admin_attr_decorator class SourceVersionAdmin(admin.ModelAdmin): readonly_fields = ("created_at",) - list_display = ["__str__", "data_source", "number", "created_at", "updated_at"] + list_display = ["id", "data_source", "number", "created_at", "updated_at"] list_filter = ["data_source", "created_at", "updated_at"] search_fields = ["data_source__name", "number", "description"] autocomplete_fields = ["data_source"] @@ -824,7 +826,7 @@ class EntityDuplicateAnalyzisAdmin(admin.ModelAdmin): class OrgUnitChangeRequestAdmin(admin.ModelAdmin): list_display = ("pk", "org_unit", "created_at", "status") list_display_links = ("pk", "org_unit") - list_filter = ("status", "kind") + list_filter = ("status", "kind", "data_source_synchronization") readonly_fields = ( "uuid", "created_at", @@ -849,6 +851,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin): "new_reference_instances", "payment", "potential_payment", + "data_source_synchronization", ) fieldsets = ( ( @@ -898,6 +901,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin): "updated_at", "updated_by", "rejection_comment", + "data_source_synchronization", ) }, ), @@ -919,7 +923,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin): ) def get_queryset(self, request): - return super().get_queryset(request).select_related("org_unit__org_unit_type") + return super().get_queryset(request).select_related("org_unit__org_unit_type", "data_source_synchronization") @admin.register(Config) @@ -1077,6 +1081,40 @@ class Media: } +@admin.register(DataSourceVersionsSynchronization) +class DataSourceVersionsSynchronizationAdmin(admin.ModelAdmin): + list_display = ( + "pk", + "name", + "account", + "created_by", + "count_create", + "count_update", + ) + list_display_links = ("pk", "name") + autocomplete_fields = ("account", "created_by", "source_version_to_update", "source_version_to_compare_with") + readonly_fields = ( + "json_diff", + "count_create", + "count_update", + "created_at", + "updated_at", + "sync_task", + ) + + def get_queryset(self, request): + return ( + super() + .get_queryset(request) + .select_related( + "source_version_to_update__data_source", + "source_version_to_compare_with__data_source", + "account", + "created_by", + ) + ) + + admin.site.register(AccountFeatureFlag) admin.site.register(Device) admin.site.register(DeviceOwnership) diff --git a/iaso/diffing/__init__.py b/iaso/diffing/__init__.py index 9eb311e36a..7bb030961d 100644 --- a/iaso/diffing/__init__.py +++ b/iaso/diffing/__init__.py @@ -2,3 +2,4 @@ from .differ import Differ from .dumper import Dumper from .exporter import Exporter +from .synchronizer import DataSourceVersionsSynchronizer, diffs_to_json diff --git a/iaso/diffing/comparisons.py b/iaso/diffing/comparisons.py index b385d9e28f..e9c0be39f0 100644 --- a/iaso/diffing/comparisons.py +++ b/iaso/diffing/comparisons.py @@ -92,7 +92,7 @@ def access(self, org_unit): groups = [] for group in org_unit.groups.all(): if group.source_ref == self.group_ref: - groups.append({"id": group.source_ref, "name": group.name}) + groups.append({"id": group.source_ref, "name": group.name, "iaso_id": group.pk}) return groups diff --git a/iaso/diffing/differ.py b/iaso/diffing/differ.py index 64cfebde31..726f384ac8 100644 --- a/iaso/diffing/differ.py +++ b/iaso/diffing/differ.py @@ -1,3 +1,5 @@ +from django.db.models import Q + from iaso.models import OrgUnit, GroupSet, Group from .comparisons import as_field_types, Diff, Comparison @@ -19,7 +21,7 @@ def __init__(self, logger): self.iaso_logger = logger def load_pyramid(self, version, validation_status=None, top_org_unit=None, org_unit_types=None): - self.iaso_logger.info("loading pyramid ", version.data_source, version, top_org_unit, org_unit_types) + self.iaso_logger.info(f"loading pyramid {version.data_source} {version} {top_org_unit} {org_unit_types}") queryset = ( OrgUnit.objects.prefetch_related("groups") .prefetch_related("groups__group_sets") @@ -62,11 +64,13 @@ def diff( field_names.append("groupset:" + group_set.source_ref + ":" + group_set.name) for group in group_set.groups.all(): groups_with_with_groupset.append(group.id) - for group in Group.objects.filter(source_version=version): + for group in Group.objects.filter(Q(source_version=version) | Q(source_version=version_ref)).distinct( + "source_ref" + ): if group.id not in groups_with_with_groupset and group.source_ref: field_names.append("group:" + group.source_ref + ":" + group.name) - self.iaso_logger.info("will compare the following fields ", field_names) + self.iaso_logger.info(f"will compare the following fields {field_names}") field_types = as_field_types(field_names) orgunits_dhis2 = self.load_pyramid( @@ -78,9 +82,7 @@ def diff( top_org_unit=top_org_unit_ref, org_unit_types=org_unit_types_ref, ) - self.iaso_logger.info( - "comparing ", version_ref, "(", len(orgunits_dhis2), ")", " and ", version, "(", len(orgunit_refs), ")" - ) + self.iaso_logger.info(f"comparing {version_ref} ({len(orgunits_dhis2)}) and {version} ({len(orgunit_refs)})") # speed how to index_by(&:source_ref) diffs = [] index = 0 @@ -146,7 +148,7 @@ def compare_fields(self, orgunit_dhis2, orgunit_ref, field_types): else: status = "modified" - if dhis2_value is None and ref_value is not None: + if not dhis2_value and ref_value: status = "new" if not same and dhis2_value is not None and (ref_value is None or ref_value == []): status = "deleted" diff --git a/iaso/diffing/dumper.py b/iaso/diffing/dumper.py index 423bba897d..665491e8c5 100644 --- a/iaso/diffing/dumper.py +++ b/iaso/diffing/dumper.py @@ -2,6 +2,8 @@ import json from django.contrib.gis.geos import GEOSGeometry +from django.core.serializers.json import DjangoJSONEncoder +from django.forms import model_to_dict from iaso.management.commands.command_logger import CommandLogger @@ -18,14 +20,19 @@ def color(status): return CommandLogger.END -class ShapelyJsonEncoder(json.JSONEncoder): - def __init__(self, **kwargs): - super(ShapelyJsonEncoder, self).__init__(**kwargs) - +class DiffJSONEncoder(DjangoJSONEncoder): def default(self, obj): - if hasattr(obj, "as_dict"): + if obj.__class__.__name__ in ["Diff", "Comparison"]: return obj.as_dict() - return obj.wkt + if obj.__class__.__name__ == "OrgUnit": + return model_to_dict(obj) + if obj.__class__.__name__ == "PathValue": + # See django_ltree.fields + # https://github.com/mariocesar/django-ltree/blob/154c7e/django_ltree/fields.py#L27-L28 + return str(obj) + if obj.__class__.__name__ == "MultiPolygon": + return obj.wkt + return super().default(obj) class Dumper: @@ -60,8 +67,11 @@ def dump_stats(self, diffs): self.iaso_logger.info(json.dumps(stats, indent=4)) return stats + def as_json(self, diffs): + return json.dumps(diffs, indent=4, cls=DiffJSONEncoder) + def dump_as_json(self, diffs): - self.iaso_logger.info(json.dumps(diffs, indent=4, cls=ShapelyJsonEncoder)) + self.iaso_logger.info(self.as_json(diffs)) def dump_as_csv(self, diffs, fields, csv_file, number_of_parents=5): res = [] diff --git a/iaso/diffing/synchronizer.py b/iaso/diffing/synchronizer.py new file mode 100644 index 0000000000..529788dec4 --- /dev/null +++ b/iaso/diffing/synchronizer.py @@ -0,0 +1,414 @@ +import datetime +import json +import logging +import uuid + +from dataclasses import dataclass +from itertools import islice +from typing import Optional + +from rest_framework.renderers import JSONRenderer + +from iaso.models import Group, OrgUnit, OrgUnitChangeRequest, DataSourceVersionsSynchronization +from .synchronizer_serializers import DataSourceVersionsSynchronizerDiffSerializer + + +logger = logging.getLogger(__name__) + + +def diffs_to_json(diffs) -> str: + serializer = DataSourceVersionsSynchronizerDiffSerializer(diffs, many=True) + return json.dumps(serializer.data) + + +@dataclass +class ChangeRequestGroups: + change_request_id: Optional[int] + org_unit_id: int + old_groups_ids: set[int] + new_groups_ids: set[int] + + +@dataclass +class OrgUnitMatching: + corresponding_id: Optional[int] + corresponding_parent_id: Optional[int] + + +class DataSourceVersionsSynchronizer: + """ + The synchronization mechanism for the `DataSourceVersionsSynchronization` model. + """ + + def __init__(self, data_source_sync: DataSourceVersionsSynchronization): + self.data_source_sync = data_source_sync + + # The JSON that we deserialized is assumed to have been serialized + # by `iaso.diffing.dumper.DataSourceVersionsSynchronizationEncoder`. + self.diffs = json.loads(data_source_sync.json_diff) + + self.change_requests_groups_to_bulk_create = {} + self.change_requests_to_bulk_create = [] + self.groups_matching = {} + self.groups_to_bulk_create = {} + self.org_units_matching = {} + self.org_units_to_bulk_create = [] + + self.insert_batch_size = 100 + self.json_batch_size = 10 + + def synchronize(self) -> None: + self._prepare_groups_matching() + self._create_missing_org_units_and_prepare_missing_groups() + self._bulk_create_missing_groups() + self._prepare_change_requests() + self._bulk_create_change_requests() + self._bulk_create_change_request_groups() + + @staticmethod + def sort_by_path(diffs: list[dict]) -> list: + sorted_list = sorted(diffs, key=lambda d: str(d["org_unit"]["path"])) + return sorted_list + + @staticmethod + def parse_date_str(date_str: str) -> datetime.date: + return datetime.datetime.strptime(date_str, "%Y-%m-%d").date() + + @staticmethod + def has_group_changes(comparisons: list[dict]) -> bool: + return any( + [ + comparison["status"] in ["new", "deleted"] + for comparison in comparisons + if comparison["field"].startswith("group:") + ] + ) + + def _prepare_groups_matching(self) -> None: + """ + Populate the `self.groups_matching` dict. + It's used to match `Group`s between pyramids based on the common `source_ref`. + It will be completed later with potential new groups. + """ + existing_groups = Group.objects.filter(source_version=self.data_source_sync.source_version_to_update).only( + "pk", "source_ref" + ) + for group in existing_groups: + if not group.source_ref: + logger.error( + f"Ignoring Group ID #{group.pk} because it has no `source_ref` attribute.", + extra={"group": group, "data_source_sync": self.data_source_sync}, + ) + continue + self.groups_matching[group.source_ref] = group.pk + + def _create_missing_org_units_and_prepare_missing_groups(self) -> None: + """ + Create missing `OrgUnit`s and groups prepare the list of missing `Group`s. + + Because of the tree structure of the `OrgUnit` model, it's hard to bulk create them. + So we sacrifice performance for the sake of simplicity by creating the missing `OrgUnit`s in a loop. + """ + # Cast the list into a generator to be able to iterate over it chunk by chunk. + missing_org_units_diff_generator = (diff for diff in self.sort_by_path(self.diffs) if diff["status"] == "new") + + while True: + # Get a subset of the generator. + batch_diff = list(islice(missing_org_units_diff_generator, self.json_batch_size)) + + if not batch_diff: + break + + # Prefetch `OrgUnit`s to avoid triggering one SQL query per loop iteration. + org_unit_ids = [diff["orgunit_ref"]["id"] for diff in batch_diff] + org_units = OrgUnit.objects.filter(pk__in=org_unit_ids).select_related("parent").prefetch_related("groups") + + for diff in batch_diff: + org_unit_id = diff["orgunit_ref"]["id"] + org_unit = next(org_unit for org_unit in org_units if org_unit.id == org_unit_id) + + if not org_unit.source_ref: + logger.error( + f"Ignoring OrgUnit ID #{org_unit.pk} because it has no `source_ref` attribute.", + extra={"org_unit": org_unit, "data_source_sync": self.data_source_sync}, + ) + continue + + corresponding_parent = None + if org_unit.parent: + # `get()` will always work here because `_sort_by_path()` is applied to the diff. + corresponding_parent = OrgUnit.objects.get( + source_ref=org_unit.parent.source_ref, + version=self.data_source_sync.source_version_to_update, + ) + + self.org_units_matching[org_unit.source_ref] = OrgUnitMatching( + corresponding_id=None, # This will be populated after the bulk creation. + corresponding_parent_id=corresponding_parent.pk if corresponding_parent else None, + ) + + for group in org_unit.groups.all(): + old_pk = group.pk + if ( + group.source_ref + and group.source_ref not in self.groups_matching.keys() + and old_pk not in self.groups_to_bulk_create.keys() + ): + # Duplicate the `Group` in the pyramid to update. + group.pk = None + group.source_version = self.data_source_sync.source_version_to_update + self.groups_to_bulk_create[old_pk] = group + + # Duplicate the `OrgUnit` in the pyramid to update. + org_unit.pk = None + org_unit.validation_status = org_unit.VALIDATION_NEW + org_unit.parent = corresponding_parent + org_unit.version = self.data_source_sync.source_version_to_update + org_unit.uuid = uuid.uuid4() + org_unit.creator = self.data_source_sync.created_by + org_unit.path = None # This will be calculated if the change request is approved. + org_unit.save(skip_calculate_path=True) + + self.org_units_matching[org_unit.source_ref].corresponding_id = org_unit.pk + + def _bulk_create_missing_groups(self) -> None: + # Cast the list into a generator to be able to iterate over it chunk by chunk. + new_groups_generator = (item for item in self.groups_to_bulk_create.values()) + + while True: + new_groups_batch = list(islice(new_groups_generator, self.json_batch_size)) + + if not new_groups_batch: + break + + new_groups = Group.objects.bulk_create(new_groups_batch, self.insert_batch_size) + for new_group in new_groups: + self.groups_matching[new_group.source_ref] = new_group.pk + + def _prepare_change_requests(self) -> None: + # Cast the list into a generator to be able to iterate over it chunk by chunk. + change_requests_diff_generator = (diff for diff in self.diffs if diff["status"] in ["new", "modified"]) + + while True: + # Get a subset of the generator. + batch_diff = list(islice(change_requests_diff_generator, self.json_batch_size)) + + if not batch_diff: + break + + for diff in batch_diff: + if diff["status"] == "new": + change_request, group_changes = self._prepare_new_change_requests(diff) + else: + change_request, group_changes = self._prepare_modified_change_requests(diff) + + if not change_request: + continue + + self.change_requests_to_bulk_create.append(change_request) + + old_groups_ids = set() + new_groups_ids = set() + for group_change in group_changes: + old_ids = [group["iaso_id"] for group in group_change["before"]] if group_change["before"] else [] + new_ids = [group["iaso_id"] for group in group_change["after"]] if group_change["after"] else [] + if group_change["status"] == "same": + old_groups_ids.update(old_ids) + new_groups_ids.update(new_ids) + elif group_change["status"] == "deleted": + old_groups_ids.update(old_ids) + elif group_change["status"] == "new": + new_groups_ids.update(new_ids) + + self.change_requests_groups_to_bulk_create[change_request.org_unit_id] = ChangeRequestGroups( + change_request_id=None, # This will be populated after the bulk creation. + org_unit_id=change_request.org_unit_id, + old_groups_ids=old_groups_ids, + new_groups_ids=new_groups_ids, + ) + + def _prepare_new_change_requests(self, diff: dict) -> tuple[Optional[OrgUnitChangeRequest], Optional[list]]: + org_unit = diff["orgunit_ref"] + requested_fields = [ + f"new_{field}" + for field in ["name", "parent", "opening_date", "closed_date"] + if org_unit.get(field) not in [None, ""] + ] + + if not requested_fields: + logger.error( + f"Ignoring OrgUnit ID #{diff['orgunit_ref']['id']} because `requested_fields` is empty.", + extra={"diff": diff, "data_source_sync": self.data_source_sync}, + ) + return None, None + + new_name = "" + if "new_name" in requested_fields: + new_name = org_unit["name"] + + new_opening_date = None + if "new_opening_date" in requested_fields: + new_opening_date = self.parse_date_str(org_unit["opening_date"]) + + new_closed_date = None + if "new_closed_date" in requested_fields: + new_closed_date = self.parse_date_str(org_unit["closed_date"]) + + group_changes = [] + if self.has_group_changes(diff["comparisons"]): + requested_fields.append("new_groups") + group_changes = [ + comparison for comparison in diff["comparisons"] if comparison["field"].startswith("group:") + ] + for group_change in group_changes: + new_groups = group_change["after"] if group_change["after"] else [] + # Find the corresponding `Group` ID in the pyramid to update. + for new_group in new_groups: + source_ref = new_group["id"] + matching_iaso_id = self.groups_matching.get(source_ref) + if matching_iaso_id: + new_group["iaso_id"] = matching_iaso_id + else: + logger.error( + f"Unable to find a corresponding `Group` with `source_ref={source_ref}` in the pyramid to update.", + extra={"new_group": new_group, "data_source_sync": self.data_source_sync}, + ) + + matching = self.org_units_matching[org_unit["source_ref"]] + + org_unit_change_request = OrgUnitChangeRequest( + # Data. + kind=OrgUnitChangeRequest.Kind.ORG_UNIT_CREATION, + created_by=self.data_source_sync.created_by, + requested_fields=requested_fields, + data_source_synchronization=self.data_source_sync, + org_unit_id=matching.corresponding_id, + # No old values because we are creating a new org unit. + old_parent_id=None, + old_name="", + old_org_unit_type_id=None, + old_location=None, + old_opening_date=None, + old_closed_date=None, + # New values. + new_parent_id=matching.corresponding_parent_id, + new_name=new_name, + new_opening_date=new_opening_date, + new_closed_date=new_closed_date, + ) + + return org_unit_change_request, group_changes + + def _prepare_modified_change_requests(self, diff: dict) -> tuple[OrgUnitChangeRequest, list]: + changes = { + comparison["field"]: comparison["after"] + for comparison in diff["comparisons"] + if comparison["status"] == "modified" + and comparison["field"] in ["name", "parent", "opening_date", "closed_date"] + } + + org_unit = diff["orgunit_dhis2"] + requested_fields = [] + new_parent = None + new_name = "" + new_opening_date = None + new_closed_date = None + + if changes.get("parent"): + new_parent = changes["parent"] + requested_fields.append("new_parent") + + if changes.get("name"): + new_name = changes["name"] + requested_fields.append("new_name") + + if changes.get("opening_date"): + new_opening_date = self.parse_date_str(changes["opening_date"]) + requested_fields.append("new_opening_date") + + if changes.get("closed_date"): + new_closed_date = self.parse_date_str(changes["closed_date"]) + requested_fields.append("new_closed_date") + + group_changes = [] + if self.has_group_changes(diff["comparisons"]): + requested_fields.append("new_groups") + group_changes = [ + comparison for comparison in diff["comparisons"] if comparison["field"].startswith("group:") + ] + + org_unit_change_request = OrgUnitChangeRequest( + # Data. + kind=OrgUnitChangeRequest.Kind.ORG_UNIT_CHANGE, + created_by=self.data_source_sync.created_by, + requested_fields=requested_fields, + data_source_synchronization=self.data_source_sync, + org_unit_id=org_unit["id"], + # Old values. + old_parent_id=org_unit.get("parent"), + old_name=org_unit.get("name", ""), + old_org_unit_type_id=org_unit.get("org_unit_type"), + old_location=org_unit.get("location"), + old_opening_date=org_unit.get("opening_date"), + old_closed_date=org_unit.get("closed_date"), + # New values. + new_parent=new_parent, + new_name=new_name, + new_opening_date=new_opening_date, + new_closed_date=new_closed_date, + ) + + return org_unit_change_request, group_changes + + def _bulk_create_change_requests(self) -> None: + # Cast the list into a generator to be able to iterate over it chunk by chunk. + change_requests_generator = (item for item in self.change_requests_to_bulk_create) + + while True: + change_requests_batch = list(islice(change_requests_generator, self.insert_batch_size)) + + if not change_requests_batch: + break + + change_requests = OrgUnitChangeRequest.objects.bulk_create(change_requests_batch, self.insert_batch_size) + + for change_request in change_requests: + groups_to_bulk_create = self.change_requests_groups_to_bulk_create.get(change_request.org_unit_id) + if groups_to_bulk_create: + # Link groups related to the newly created change request. + groups_to_bulk_create.change_request_id = change_request.pk + + def _bulk_create_change_request_groups(self) -> None: + """ + Use the "through" table to bulk update old and new groups (m2m). + """ + old_groups = [] + new_groups = [] + + for groups_info in self.change_requests_groups_to_bulk_create.values(): + for old_group_id in groups_info.old_groups_ids: + old_groups.append( + OrgUnitChangeRequest.old_groups.through( + orgunitchangerequest_id=groups_info.change_request_id, group_id=old_group_id + ) + ) + for new_group_id in groups_info.new_groups_ids: + new_groups.append( + OrgUnitChangeRequest.new_groups.through( + orgunitchangerequest_id=groups_info.change_request_id, group_id=new_group_id + ) + ) + + old_groups_generator = (group for group in old_groups) + while True: + old_groups_batch = list(islice(old_groups_generator, self.insert_batch_size)) + if not old_groups_batch: + break + OrgUnitChangeRequest.old_groups.through.objects.bulk_create(old_groups_batch, self.insert_batch_size) + + new_groups_generator = (group for group in new_groups) + while True: + new_groups_batch = list(islice(new_groups_generator, self.insert_batch_size)) + if not new_groups_batch: + break + OrgUnitChangeRequest.new_groups.through.objects.bulk_create(new_groups_batch, self.insert_batch_size) diff --git a/iaso/diffing/synchronizer_serializers.py b/iaso/diffing/synchronizer_serializers.py new file mode 100644 index 0000000000..c35dc287d9 --- /dev/null +++ b/iaso/diffing/synchronizer_serializers.py @@ -0,0 +1,81 @@ +from rest_framework import serializers + +from iaso.models import OrgUnit + + +class BaseComparisonSerializer(serializers.Serializer): + field = serializers.CharField(read_only=True) + before = serializers.CharField(read_only=True) + after = serializers.CharField(read_only=True) + status = serializers.CharField(read_only=True) + distance = serializers.IntegerField(read_only=True) + + +class NameComparisonSerializer(BaseComparisonSerializer): + pass + + +class ParentComparisonSerializer(BaseComparisonSerializer): + pass + + +class DateComparisonSerializer(BaseComparisonSerializer): + before = serializers.DateField(read_only=True) + after = serializers.DateField(read_only=True) + + +class GroupComparisonSerializer(BaseComparisonSerializer): + before = serializers.ListField(read_only=True) + after = serializers.ListField(read_only=True) + + +class GeometryComparisonSerializer(BaseComparisonSerializer): + before = serializers.CharField(source="before.wkt", read_only=True) + after = serializers.CharField(source="after.wkt", read_only=True) + + +class DiffOrgUnitSerializer(serializers.ModelSerializer): + class Meta: + model = OrgUnit + fields = [ + "id", + "version", + "source_ref", + "location", + "org_unit_type", + "path", + # Fields that can be synchronized via change requests. + "name", + "parent", + "opening_date", + "closed_date", + "groups", + ] + + +class DataSourceVersionsSynchronizerDiffSerializer(serializers.Serializer): + """ + This serializer generates the exact format of the JSON stored in the + `DataSourceVersionsSynchronization.json_diff` field. + """ + + org_unit = DiffOrgUnitSerializer() + orgunit_ref = DiffOrgUnitSerializer() + orgunit_dhis2 = DiffOrgUnitSerializer() + status = serializers.CharField() + comparisons = serializers.SerializerMethodField("get_comparisons_serializer") + + def get_comparisons_serializer(self, obj): + serializers = [] + for comparison in obj.comparisons: + if comparison.field == "name": + serializers.append(NameComparisonSerializer(comparison).data) + if comparison.field == "parent": + serializers.append(ParentComparisonSerializer(comparison).data) + if comparison.field in ("opening_date", "closed_date"): + serializers.append(DateComparisonSerializer(comparison).data) + if comparison.field == "geometry": + serializers.append(GeometryComparisonSerializer(comparison).data) + if comparison.field.startswith("group:"): + serializers.append(GroupComparisonSerializer(comparison).data) + return serializers diff --git a/iaso/migrations/0314_datasourceversionssynchronization_and_more.py b/iaso/migrations/0314_datasourceversionssynchronization_and_more.py new file mode 100644 index 0000000000..ba8adb6490 --- /dev/null +++ b/iaso/migrations/0314_datasourceversionssynchronization_and_more.py @@ -0,0 +1,108 @@ +# Generated by Django 4.2.17 on 2025-01-09 14:03 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("iaso", "0313_page_superset_dashboard_id_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="DataSourceVersionsSynchronization", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ( + "name", + models.CharField( + help_text="Used in the UI e.g. to filter Change Requests by Data Source Synchronization operations.", + max_length=255, + ), + ), + ( + "json_diff", + models.JSONField(blank=True, help_text="The diff used to create change requests.", null=True), + ), + ( + "diff_config", + models.TextField( + blank=True, help_text="A string representation of the parameters used for the diff." + ), + ), + ( + "count_create", + models.PositiveIntegerField( + default=0, + help_text="The number of change requests that will be generated to create an org unit.", + ), + ), + ( + "count_update", + models.PositiveIntegerField( + default=0, + help_text="The number of change requests that will be generated to update an org unit.", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("account", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="iaso.account")), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_data_source_synchronizations", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "source_version_to_compare_with", + models.ForeignKey( + help_text="The version of the pyramid to use as a comparison.", + on_delete=django.db.models.deletion.CASCADE, + related_name="synchronized_as_source_version_to_compare_with", + to="iaso.sourceversion", + ), + ), + ( + "source_version_to_update", + models.ForeignKey( + help_text="The version of the pyramid for which we want to generate change requests.", + on_delete=django.db.models.deletion.CASCADE, + related_name="synchronized_as_source_version_to_update", + to="iaso.sourceversion", + ), + ), + ( + "sync_task", + models.OneToOneField( + blank=True, + help_text="The background task that used the diff to create change requests.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="iaso.task", + ), + ), + ], + options={ + "verbose_name": "Data source synchronization", + }, + ), + migrations.AddField( + model_name="orgunitchangerequest", + name="data_source_synchronization", + field=models.ForeignKey( + blank=True, + help_text="The data source synchronization that generated this change request.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="change_requests", + to="iaso.datasourceversionssynchronization", + ), + ), + ] diff --git a/iaso/models/__init__.py b/iaso/models/__init__.py index a1390def44..d90bab8396 100644 --- a/iaso/models/__init__.py +++ b/iaso/models/__init__.py @@ -1,5 +1,6 @@ from .base import * from .base import Instance +from .data_source import DataSource, DataSourceVersionsSynchronization, SourceVersion from .device import Device, DeviceOwnership, DevicePosition from .forms import Form, FormVersion, FormPredefinedFilter, FormAttachment from .org_unit import OrgUnit, OrgUnitType, OrgUnitChangeRequest diff --git a/iaso/models/data_source.py b/iaso/models/data_source.py index c1272ba2bf..f5b23153cf 100644 --- a/iaso/models/data_source.py +++ b/iaso/models/data_source.py @@ -1,7 +1,18 @@ +import logging +import typing + +from django.contrib.auth.models import AnonymousUser, User from django.contrib.postgres.fields import ArrayField +from django.core.exceptions import ValidationError from django.db import models -from django.contrib.auth.models import AnonymousUser, User -import typing +from django.utils.translation import gettext_lazy as _ + + +if typing.TYPE_CHECKING: + from iaso.models import OrgUnit, OrgUnitType + + +logger = logging.getLogger(__name__) class DataSource(models.Model): @@ -31,7 +42,7 @@ class DataSource(models.Model): public = models.BooleanField(default=False) def __str__(self): - return "%s " % (self.name,) + return f"#{self.pk} {self.name}" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -104,7 +115,7 @@ class Meta: ] def __str__(self): - return "%s %d" % (self.data_source, self.number) + return f"#{self.pk} n°{self.number} - Data source: {self.data_source}" def as_dict(self): return { @@ -161,3 +172,188 @@ def as_report_dict(self): group_report[name] = self.orgunit_set.filter(groups__id=ident).count() report["groups"] = group_report return report + + +class DataSourceVersionsSynchronization(models.Model): + """ + This table allows to synchronize two pyramids by creating "change requests" + based on their diff. + + The logic is tightly coupled to the `iaso.diffing` module. + + Fields that can be synchronized: + + ["name", "parent", "opening_date", "closed_date", "groups"] + + Basic business use case: + + - often, a pyramid of org units is created in DHIS2 and imported in IASO + - IASO is then used to update the pyramid + - but meanwhile, people may continue to make changes in DHIS2 + - as a consequence, the two pyramids diverge + - so we need to synchronize the changes in the two pyramids + + The synchronization is done in two steps: + + 1. `self.create_json_diff()` + - this will compute the diff between the two pyramids + - the user will now how many change requests will be created + - he still has the choice of giving up if there are too many differences + 2. `self.synchronize_source_versions()` + - if the user is OK, this will synchronize the changes by creating change requests + + """ + + name = models.CharField( + max_length=255, + help_text=_("Used in the UI e.g. to filter Change Requests by Data Source Synchronization operations."), + ) + source_version_to_update = models.ForeignKey( + SourceVersion, + on_delete=models.CASCADE, + related_name="synchronized_as_source_version_to_update", + help_text=_("The version of the pyramid for which we want to generate change requests."), + ) + source_version_to_compare_with = models.ForeignKey( + SourceVersion, + on_delete=models.CASCADE, + related_name="synchronized_as_source_version_to_compare_with", + help_text=_("The version of the pyramid to use as a comparison."), + ) + + # The JSON format is defined in `iaso.diffing.synchronizer_serializers.DataSourceVersionsSynchronizerDiffSerializer`. + json_diff = models.JSONField(null=True, blank=True, help_text=_("The diff used to create change requests.")) + diff_config = models.TextField( + blank=True, help_text=_("A string representation of the parameters used for the diff.") + ) + count_create = models.PositiveIntegerField( + default=0, help_text=_("The number of change requests that will be generated to create an org unit.") + ) + count_update = models.PositiveIntegerField( + default=0, help_text=_("The number of change requests that will be generated to update an org unit.") + ) + + sync_task = models.OneToOneField( + "Task", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="+", + help_text=_("The background task that used the diff to create change requests."), + ) + + # Metadata. + account = models.ForeignKey("Account", on_delete=models.CASCADE) + created_by = models.ForeignKey( + User, null=True, on_delete=models.SET_NULL, related_name="created_data_source_synchronizations" + ) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = _("Data source synchronization") + + def __str__(self) -> str: + return self.name + + def save(self, *args, **kwargs): + self.clean() + super().save(*args, **kwargs) + + def clean(self, *args, **kwargs): + super().clean() + self.clean_data_source_versions() + + def clean_data_source_versions(self) -> None: + if self.source_version_to_update.data_source_id != self.source_version_to_compare_with.data_source_id: + raise ValidationError("The two versions to compare must be linked to the same data source.") + if self.source_version_to_update.pk == self.source_version_to_compare_with.pk: + raise ValidationError("The two versions to compare must be different.") + + def create_json_diff( + self, + logger_to_use: logging.Logger = None, + source_version_to_update_validation_status: str = None, + source_version_to_update_top_org_unit: "OrgUnit" = None, + source_version_to_update_org_unit_types: list["OrgUnitType"] = None, + source_version_to_compare_with_validation_status: str = None, + source_version_to_compare_with_top_org_unit: "OrgUnit" = None, + source_version_to_compare_with_org_unit_types: list["OrgUnitType"] = None, + ignore_groups: bool = False, + show_deleted_org_units: bool = False, + field_names: list[str] = None, + ) -> None: + # Prevent a circular import. + from iaso.diffing import Differ, diffs_to_json + + if self.change_requests.exists(): + raise ValidationError("Change requests have already been created.") + + differ_params = { + # Version to update. + "version": self.source_version_to_update, + "validation_status": source_version_to_update_validation_status, + "top_org_unit": source_version_to_update_top_org_unit, + "org_unit_types": source_version_to_update_org_unit_types, + # Version to compare with. + "version_ref": self.source_version_to_compare_with, + "validation_status_ref": source_version_to_compare_with_validation_status, + "top_org_unit_ref": source_version_to_compare_with_top_org_unit, + "org_unit_types_ref": source_version_to_compare_with_org_unit_types, + # Options. + "ignore_groups": ignore_groups, + "show_deleted_org_units": show_deleted_org_units, + "field_names": field_names, + } + diffs, _ = Differ(logger_to_use or logger).diff(**differ_params) + + # Reduce the size of the diff that will be stored in the DB. + diffs = [diff for diff in diffs if diff.status != "same"] + + count_status = { + "new": 0, + "modified": 0, + } + for diff in diffs: + if diff.status in count_status: + count_status[diff.status] += 1 + + # Keep track of the parameters used for the diff. + differ_config = { + # Version to update. + "version": self.source_version_to_update.pk, + "validation_status": source_version_to_update_validation_status, + "top_org_unit": source_version_to_update_top_org_unit.pk if source_version_to_update_top_org_unit else None, + "org_unit_types": [out.pk for out in source_version_to_update_org_unit_types] + if source_version_to_update_org_unit_types + else None, + # Version to compare with. + "version_ref": self.source_version_to_compare_with.pk, + "validation_status_ref": source_version_to_compare_with_validation_status, + "top_org_unit_ref": source_version_to_compare_with_top_org_unit.pk + if source_version_to_compare_with_top_org_unit + else None, + "org_unit_types_ref": [out.pk for out in source_version_to_compare_with_org_unit_types] + if source_version_to_compare_with_org_unit_types + else None, + # Options. + "ignore_groups": ignore_groups, + "show_deleted_org_units": show_deleted_org_units, + "field_names": field_names, + } + + self.count_create = count_status["new"] + self.count_update = count_status["modified"] + self.json_diff = diffs_to_json(diffs) + self.diff_config = str(differ_config) + self.save() + + def synchronize_source_versions(self): + # Prevent a circular import. + from iaso.diffing import DataSourceVersionsSynchronizer + + if self.change_requests.exists(): + raise ValidationError("Change requests have already been created.") + + synchronizer = DataSourceVersionsSynchronizer(data_source_sync=self) + synchronizer.synchronize() diff --git a/iaso/models/org_unit.py b/iaso/models/org_unit.py index b884122494..171f174b77 100644 --- a/iaso/models/org_unit.py +++ b/iaso/models/org_unit.py @@ -1,6 +1,8 @@ +import logging import operator import typing import uuid + from copy import deepcopy from functools import reduce @@ -24,10 +26,11 @@ from ..utils.models.common import get_creator_name from .project import Project -try: # for typing - from .base import Account -except: - pass + +if typing.TYPE_CHECKING: + from iaso.models import Account + +logger = logging.getLogger(__name__) def get_or_create_org_unit_type(name: str, depth: int, account: "Account", preferred_project: Project) -> "OrgUnitType": @@ -123,7 +126,7 @@ class OrgUnitType(models.Model): updated_at = models.DateTimeField(auto_now=True) category = models.CharField(max_length=8, choices=CATEGORIES, null=True, blank=True) sub_unit_types = models.ManyToManyField("OrgUnitType", related_name="super_types", blank=True) - # Allow the creation of these sub org unit types only for mobile (IA-2153)" + # Allow the creation of these sub org unit types only for mobile (IA-2153) allow_creating_sub_unit_types = models.ManyToManyField("OrgUnitType", related_name="create_types", blank=True) reference_forms = models.ManyToManyField("Form", related_name="reference_of_org_unit_types", blank=True) projects = models.ManyToManyField("Project", related_name="unit_types", blank=False) @@ -132,7 +135,7 @@ class OrgUnitType(models.Model): objects = OrgUnitTypeManager() def __str__(self): - return "%s" % self.name + return f"#{self.pk} {self.name}" def as_dict(self, sub_units=True, app_id=None): res = { @@ -312,6 +315,9 @@ class Meta: models.Index(fields=["source_created_at"]), ] + def __str__(self) -> str: + return f"#{self.pk} {self.name}" + @property def source_created_at_with_fallback(self): return self.source_created_at if self.source_created_at else self.created_at @@ -384,9 +390,6 @@ def calculate_paths(self, force_recalculate: bool = False) -> typing.List["OrgUn return updated_records - def __str__(self): - return "%s %s %d" % (self.org_unit_type, self.name, self.id if self.id else -1) - def as_dict_for_mobile_lite(self): return { "n": self.name, @@ -719,6 +722,15 @@ class Statuses(models.TextChoices): "PotentialPayment", on_delete=models.SET_NULL, null=True, blank=True, related_name="change_requests" ) + data_source_synchronization = models.ForeignKey( + "DataSourceVersionsSynchronization", + null=True, + blank=True, + on_delete=models.CASCADE, + related_name="change_requests", + help_text="The data source synchronization that generated this change request.", + ) + objects = models.Manager.from_queryset(OrgUnitChangeRequestQuerySet)() class Meta: diff --git a/iaso/tests/api/test_orgunits.py b/iaso/tests/api/test_orgunits.py index d51446ad8b..728a41d8fb 100644 --- a/iaso/tests/api/test_orgunits.py +++ b/iaso/tests/api/test_orgunits.py @@ -813,11 +813,10 @@ def test_create_org_unit_group_ok_same_version(self): } ) ou = m.OrgUnit.objects.get(id=jr["id"]) - self.assertQuerySetEqual( - ou.groups.all().order_by("name"), - ["", ""], - transform=repr, - ) + ou_groups = ou.groups.all().order_by("name") + self.assertEqual(len(ou_groups), 2) + self.assertEqual(ou_groups[0].name, group_1.name) + self.assertEqual(ou_groups[1].name, group_2.name) def test_create_org_unit_with_reference_instance(self): self.client.force_authenticate(self.yoda) @@ -892,9 +891,10 @@ def test_edit_org_unit_retrieve_put(self): self.assertEqual(response.data["reference_instances"], []) self.assertCreated({Modification: 1}) ou = m.OrgUnit.objects.get(id=jr["id"]) - self.assertQuerySetEqual( - ou.groups.all().order_by("name"), [""], transform=repr - ) + ou_groups = ou.groups.all().order_by("name") + self.assertEqual(ou_groups.count(), 1) + self.assertEqual(ou_groups.first().name, self.elite_group.name) + self.assertEqual(ou_groups.first().source_version, self.sw_version_1) self.assertEqual(ou.id, old_ou.id) self.assertEqual(ou.name, old_ou.name) self.assertEqual(ou.parent, old_ou.parent) @@ -1118,10 +1118,16 @@ def test_edit_org_unit_edit_bad_group_fail(self): self.assertJSONResponse(response, 200) self.assertCreated({Modification: 1}) ou = m.OrgUnit.objects.get(id=old_ou.id) + # Verify group was not modified but the rest was modified - self.assertQuerySetEqual( - ou.groups.all().order_by("name"), ["", ""], transform=repr - ) + ou_groups = ou.groups.all().order_by("name") + self.assertEqual(ou_groups.count(), 2) + self.assertEqual(ou_groups[0].name, "") + self.assertEqual(ou_groups[0].source_version.data_source.name, "Evil Empire") + self.assertEqual(ou_groups[0].source_version.number, 1) + self.assertEqual(ou_groups[1].name, "bad") + self.assertEqual(ou_groups[1].source_version, None) + self.assertEqual(ou.id, old_ou.id) self.assertEqual(ou.name, "new name") self.assertEqual(ou.parent, old_ou.parent) diff --git a/iaso/tests/diffing/__init__.py b/iaso/tests/diffing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/iaso/tests/diffing/test_differ.py b/iaso/tests/diffing/test_differ.py new file mode 100644 index 0000000000..11103d9be9 --- /dev/null +++ b/iaso/tests/diffing/test_differ.py @@ -0,0 +1,324 @@ +import datetime +import logging + +import time_machine + +from django.contrib.gis.geos import MultiPolygon, Polygon + +from iaso.diffing import Differ +from iaso.tests.diffing.utils import PyramidBaseTest + + +test_logger = logging.getLogger(__name__) + + +DT = datetime.datetime(2024, 11, 28, 17, 0, 0, 0, tzinfo=datetime.timezone.utc) + + +@time_machine.travel(DT, tick=False) +class DifferTestCase(PyramidBaseTest): + """ + Test Differ. + """ + + def test_full_python_diff(self): + """ + Test that the full diff works as expected. + """ + + new_multi_polygon = MultiPolygon(Polygon([[-1.3, 2.5], [-1.7, 2.8], [-1.1, 4.1], [-1.3, 2.5]])) + + self.angola_country_to_compare_with.name = "Angola new" + self.angola_country_to_compare_with.geom = new_multi_polygon + self.angola_country_to_compare_with.opening_date = datetime.date(2022, 12, 28) + self.angola_country_to_compare_with.closed_date = datetime.date(2025, 12, 28) + self.angola_country_to_compare_with.save() + + self.angola_region_to_compare_with.name = "Huila new" + self.angola_region_to_compare_with.geom = new_multi_polygon + self.angola_region_to_compare_with.opening_date = datetime.date(2022, 12, 28) + self.angola_region_to_compare_with.closed_date = datetime.date(2025, 12, 28) + self.angola_region_to_compare_with.save() + + self.angola_district_to_compare_with.name = "Cuvango new" + self.angola_district_to_compare_with.parent = self.angola_country_to_compare_with + self.angola_district_to_compare_with.geom = new_multi_polygon + self.angola_district_to_compare_with.opening_date = datetime.date(2022, 12, 28) + self.angola_district_to_compare_with.closed_date = datetime.date(2025, 12, 28) + self.angola_district_to_compare_with.save() + + diffs, fields = Differ(test_logger).diff( + # Version to update. + version=self.source_version_to_update, + validation_status=None, + top_org_unit=None, + org_unit_types=None, + # Version to compare with. + version_ref=self.source_version_to_compare_with, + validation_status_ref=None, + top_org_unit_ref=None, + org_unit_types_ref=None, + # Options. + ignore_groups=False, + show_deleted_org_units=False, + field_names=["name", "parent", "geometry", "opening_date", "closed_date"], + ) + + self.assertEqual(len(diffs), 3) + + country_diff = next((diff for diff in diffs if diff.org_unit.org_unit_type == self.org_unit_type_country), None) + self.assertEqual(country_diff.status, "modified") + country_diff_comparisons = [comparison.as_dict() for comparison in country_diff.comparisons] + self.assertEqual(8, len(country_diff_comparisons)) + self.assertDictEqual( + country_diff_comparisons[0], + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + country_diff_comparisons[1], + { + "field": "parent", + "before": None, + "after": None, + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + country_diff_comparisons[2], + { + "field": "geometry", + "before": self.multi_polygon, + "after": new_multi_polygon, + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + country_diff_comparisons[3], + { + "field": "opening_date", + "before": datetime.date(2022, 11, 28), + "after": datetime.date(2022, 12, 28), + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + country_diff_comparisons[4], + { + "field": "closed_date", + "before": datetime.date(2025, 11, 28), + "after": datetime.date(2025, 12, 28), + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + country_diff_comparisons[5], + { + "field": "group:group-a:Group A", + "before": [{"id": "group-a", "name": "Group A", "iaso_id": self.group_a1.pk}], + "after": [{"id": "group-a", "name": "Group A", "iaso_id": self.group_a2.pk}], + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + country_diff_comparisons[6], + { + "field": "group:group-b:Group B", + "before": [{"id": "group-b", "name": "Group B", "iaso_id": self.group_b.pk}], + "after": [], + "status": "deleted", + "distance": None, + }, + ) + self.assertDictEqual( + country_diff_comparisons[7], + { + "field": "group:group-c:Group C", + "before": [], + "after": [{"id": "group-c", "name": "Group C", "iaso_id": self.group_c.pk}], + "status": "new", + "distance": None, + }, + ) + + region_diff = next((diff for diff in diffs if diff.org_unit.org_unit_type == self.org_unit_type_region), None) + self.assertEqual(region_diff.status, "modified") + region_diff_comparisons = [comparison.as_dict() for comparison in region_diff.comparisons] + self.assertEqual(8, len(region_diff_comparisons)) + self.assertDictEqual( + region_diff_comparisons[0], + { + "field": "name", + "before": "Huila", + "after": "Huila new", + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + region_diff_comparisons[1], + { + "field": "parent", + "before": "id-1", + "after": "id-1", + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + region_diff_comparisons[2], + { + "field": "geometry", + "before": self.multi_polygon, + "after": new_multi_polygon, + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + region_diff_comparisons[3], + { + "field": "opening_date", + "before": datetime.date(2022, 11, 28), + "after": datetime.date(2022, 12, 28), + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + region_diff_comparisons[4], + { + "field": "closed_date", + "before": datetime.date(2025, 11, 28), + "after": datetime.date(2025, 12, 28), + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + region_diff_comparisons[5], + { + "field": "group:group-a:Group A", + "before": [], + "after": [], + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + region_diff_comparisons[6], + { + "field": "group:group-b:Group B", + "before": [], + "after": [], + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + region_diff_comparisons[7], + { + "field": "group:group-c:Group C", + "before": [], + "after": [], + "status": "same", + "distance": 0, + }, + ) + + district_diff = next( + (diff for diff in diffs if diff.org_unit.org_unit_type == self.org_unit_type_district), None + ) + self.assertEqual(district_diff.status, "modified") + district_diff_comparisons = [comparison.as_dict() for comparison in district_diff.comparisons] + self.assertEqual(8, len(district_diff_comparisons)) + self.assertDictEqual( + district_diff_comparisons[0], + { + "field": "name", + "before": "Cuvango", + "after": "Cuvango new", + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + district_diff_comparisons[1], + { + "field": "parent", + "before": "id-2", + "after": "id-1", + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + district_diff_comparisons[2], + { + "field": "geometry", + "before": self.multi_polygon, + "after": new_multi_polygon, + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + district_diff_comparisons[3], + { + "field": "opening_date", + "before": datetime.date(2022, 11, 28), + "after": datetime.date(2022, 12, 28), + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + district_diff_comparisons[4], + { + "field": "closed_date", + "before": datetime.date(2025, 11, 28), + "after": datetime.date(2025, 12, 28), + "status": "modified", + "distance": None, + }, + ) + self.assertDictEqual( + district_diff_comparisons[5], + { + "field": "group:group-a:Group A", + "before": [], + "after": [], + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + district_diff_comparisons[6], + { + "field": "group:group-b:Group B", + "before": [], + "after": [], + "status": "same", + "distance": 0, + }, + ) + self.assertDictEqual( + region_diff_comparisons[7], + { + "field": "group:group-c:Group C", + "before": [], + "after": [], + "status": "same", + "distance": 0, + }, + ) diff --git a/iaso/tests/diffing/test_dumper.py b/iaso/tests/diffing/test_dumper.py new file mode 100644 index 0000000000..885c746fe7 --- /dev/null +++ b/iaso/tests/diffing/test_dumper.py @@ -0,0 +1,253 @@ +import datetime +import logging + +import time_machine + +from iaso.diffing import Differ, Dumper +from iaso.tests.diffing.utils import PyramidBaseTest + + +test_logger = logging.getLogger(__name__) + + +DT = datetime.datetime(2024, 11, 30, 10, 0, 0, 0, tzinfo=datetime.timezone.utc) + + +@time_machine.travel(DT, tick=False) +class DumperTestCase(PyramidBaseTest): + """ + Test Dumper. + """ + + def test_dump_as_json_for_partial_update(self): + """ + Test `Dumper.as_json()` for a partial update. + """ + # Change the name. + self.angola_country_to_compare_with.name = "Angola new" + self.angola_country_to_compare_with.save() + + # Limit the diff size with restrictions on `field_names` and `org_unit_types_ref`. + diffs, fields = Differ(test_logger).diff( + # Version to update. + version=self.source_version_to_update, + validation_status=None, + top_org_unit=None, + org_unit_types=[self.org_unit_type_country], + # Version to compare with. + version_ref=self.source_version_to_compare_with, + validation_status_ref=None, + top_org_unit_ref=None, + org_unit_types_ref=[self.org_unit_type_country], + # Options. + ignore_groups=True, + show_deleted_org_units=False, + field_names=["name"], + ) + + dumper = Dumper(test_logger) + json_diffs = dumper.as_json(diffs) + + expected_json_diffs = [ + { + "org_unit": { + "id": self.angola_country_to_compare_with.pk, + "name": "Angola new", + "uuid": None, + "custom": False, + "validated": True, + "validation_status": "VALID", + "version": self.source_version_to_compare_with.pk, + "parent": None, + "path": str(self.angola_country_to_compare_with.path), + "aliases": None, + "org_unit_type": self.org_unit_type_country.pk, + "sub_source": None, + "source_ref": "id-1", + "geom": "MULTIPOLYGON (((0 0, 0 1, 1 1, 0 0)))", + "simplified_geom": None, + "catchment": None, + "geom_ref": None, + "gps_source": None, + "location": None, + "source_created_at": None, + "creator": None, + "extra_fields": {}, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "default_image": None, + "reference_instances": [], + }, + "orgunit_ref": { + "id": self.angola_country_to_compare_with.pk, + "name": "Angola new", + "uuid": None, + "custom": False, + "validated": True, + "validation_status": "VALID", + "version": self.source_version_to_compare_with.pk, + "parent": None, + "path": str(self.angola_country_to_compare_with.path), + "aliases": None, + "org_unit_type": self.org_unit_type_country.pk, + "sub_source": None, + "source_ref": "id-1", + "geom": "MULTIPOLYGON (((0 0, 0 1, 1 1, 0 0)))", + "simplified_geom": None, + "catchment": None, + "geom_ref": None, + "gps_source": None, + "location": None, + "source_created_at": None, + "creator": None, + "extra_fields": {}, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "default_image": None, + "reference_instances": [], + }, + "orgunit_dhis2": { + "id": self.angola_country_to_update.pk, + "name": "Angola", + "uuid": None, + "custom": False, + "validated": True, + "validation_status": "VALID", + "version": self.source_version_to_update.pk, + "parent": None, + "path": str(self.angola_country_to_update.path), + "aliases": None, + "org_unit_type": self.org_unit_type_country.pk, + "sub_source": None, + "source_ref": "id-1", + "geom": "MULTIPOLYGON (((0 0, 0 1, 1 1, 0 0)))", + "simplified_geom": None, + "catchment": None, + "geom_ref": None, + "gps_source": None, + "location": None, + "source_created_at": None, + "creator": None, + "extra_fields": {}, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "default_image": None, + "reference_instances": [], + }, + "status": "modified", + "comparisons": [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + } + ], + } + ] + + self.assertJSONEqual(json_diffs, expected_json_diffs) + + def test_dump_as_json_for_a_new_org_unit(self): + """ + Test `Dumper.as_json()` for a new org unit. + """ + + # Simulate an org unit existing only in one pyramid. + self.angola_country_to_update.delete() + + # Limit the diff size with restrictions on `field_names` and `org_unit_types_ref`. + diffs, fields = Differ(test_logger).diff( + # Version to update. + version=self.source_version_to_update, + validation_status=None, + top_org_unit=None, + org_unit_types=[self.org_unit_type_country], + # Version to compare with. + version_ref=self.source_version_to_compare_with, + validation_status_ref=None, + top_org_unit_ref=None, + org_unit_types_ref=[self.org_unit_type_country], + # Options. + ignore_groups=True, + show_deleted_org_units=False, + field_names=["name"], + ) + + dumper = Dumper(test_logger) + json_diffs = dumper.as_json(diffs) + + expected_json_diffs = [ + { + "org_unit": { + "id": self.angola_country_to_compare_with.pk, + "name": "Angola", + "uuid": None, + "custom": False, + "validated": True, + "validation_status": "VALID", + "version": self.source_version_to_compare_with.pk, + "parent": None, + "path": str(self.angola_country_to_compare_with.path), + "aliases": None, + "org_unit_type": self.org_unit_type_country.pk, + "sub_source": None, + "source_ref": "id-1", + "geom": "MULTIPOLYGON (((0 0, 0 1, 1 1, 0 0)))", + "simplified_geom": None, + "catchment": None, + "geom_ref": None, + "gps_source": None, + "location": None, + "source_created_at": None, + "creator": None, + "extra_fields": {}, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "default_image": None, + "reference_instances": [], + }, + "orgunit_ref": { + "id": self.angola_country_to_compare_with.pk, + "name": "Angola", + "uuid": None, + "custom": False, + "validated": True, + "validation_status": "VALID", + "version": self.source_version_to_compare_with.pk, + "parent": None, + "path": str(self.angola_country_to_compare_with.path), + "aliases": None, + "org_unit_type": self.org_unit_type_country.pk, + "sub_source": None, + "source_ref": "id-1", + "geom": "MULTIPOLYGON (((0 0, 0 1, 1 1, 0 0)))", + "simplified_geom": None, + "catchment": None, + "geom_ref": None, + "gps_source": None, + "location": None, + "source_created_at": None, + "creator": None, + "extra_fields": {}, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "default_image": None, + "reference_instances": [], + }, + "orgunit_dhis2": None, + "status": "new", + "comparisons": [ + { + "field": "name", + "before": None, + "after": "Angola", + "status": "new", + "distance": None, + } + ], + } + ] + + self.assertJSONEqual(json_diffs, expected_json_diffs) diff --git a/iaso/tests/diffing/test_synchronizer.py b/iaso/tests/diffing/test_synchronizer.py new file mode 100644 index 0000000000..1573af3e32 --- /dev/null +++ b/iaso/tests/diffing/test_synchronizer.py @@ -0,0 +1,295 @@ +import datetime +import logging + +import time_machine + +from django.test import TestCase + +from iaso.diffing import DataSourceVersionsSynchronizer, Differ, diffs_to_json +from iaso.tests.diffing.utils import PyramidBaseTest + + +test_logger = logging.getLogger(__name__) + + +DT = datetime.datetime(2024, 11, 30, 10, 0, 0, 0, tzinfo=datetime.timezone.utc) + + +class DataSourceVersionsSynchronizerTestCase(TestCase): + """ + Test DataSourceVersionsSynchronizer. + + See also `DataSourceVersionsSynchronizationModelTestCase`. + """ + + def test_sort_by_path(self): + level1 = { + "org_unit": { + "id": 1, + "path": "99280", + }, + } + level2 = { + "org_unit": { + "id": 2, + "path": "99280.99931", + }, + } + level3 = { + "org_unit": { + "id": 3, + "path": "99280.99931.104415", + }, + } + levelNone = { + "org_unit": { + "id": 3, + "path": None, + }, + } + diffs = [level1, level3, levelNone, level2] + sorted_diffs = DataSourceVersionsSynchronizer.sort_by_path(diffs) + expected_sorted_diffs = [level1, level2, level3, levelNone] + self.assertEqual(sorted_diffs, expected_sorted_diffs) + + def test_parse_date_str(self): + parsed_date = DataSourceVersionsSynchronizer.parse_date_str("2025-11-28") + expected_parsed_date = datetime.date(2025, 11, 28) + self.assertEqual(parsed_date, expected_parsed_date) + + def test_has_group_changes(self): + # No group change: `has_group_changes()` should be False. + comparisons = [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + }, + ] + has_group_changes = DataSourceVersionsSynchronizer.has_group_changes(comparisons) + self.assertFalse(has_group_changes) + + # Group is the same: `has_group_changes()` should be False. + comparisons = [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + }, + { + "field": "group:group-a:Group A", + "before": [{"id": "group-a", "name": "Group A", "iaso_id": 1260}], + "after": [{"id": "group-a", "name": "Group A", "iaso_id": 1262}], + "status": "same", + "distance": 0, + }, + ] + has_group_changes = DataSourceVersionsSynchronizer.has_group_changes(comparisons) + self.assertFalse(has_group_changes) + + # A group is deleted: `has_group_changes()` should be True. + comparisons = [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + }, + { + "field": "group:group-b:Group B", + "before": [{"id": "group-b", "name": "Group B", "iaso_id": 1261}], + "after": [], + "status": "deleted", + "distance": None, + }, + ] + has_group_changes = DataSourceVersionsSynchronizer.has_group_changes(comparisons) + self.assertTrue(has_group_changes) + + # A group is created: `has_group_changes()` should be True. + comparisons = [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + }, + { + "field": "group:group-c:Group C", + "before": [], + "after": [{"id": "group-c", "name": "Group C", "iaso_id": 1263}], + "status": "new", + "distance": None, + }, + ] + has_group_changes = DataSourceVersionsSynchronizer.has_group_changes(comparisons) + self.assertTrue(has_group_changes) + + +@time_machine.travel(DT, tick=False) +class DiffsToJsonTestCase(PyramidBaseTest): + """ + Test `diffs_to_json()`. + """ + + def test_dump_as_json_for_org_unit_update(self): + """ + Test the format of `Dumper.as_json()` for a modified org unit. + """ + # Change the name. + self.angola_country_to_compare_with.name = "Angola new" + self.angola_country_to_compare_with.save() + + # Limit the diff size with restrictions on `field_names` and `org_unit_types_ref`. + diffs, fields = Differ(test_logger).diff( + # Version to update. + version=self.source_version_to_update, + validation_status=None, + top_org_unit=None, + org_unit_types=[self.org_unit_type_country], + # Version to compare with. + version_ref=self.source_version_to_compare_with, + validation_status_ref=None, + top_org_unit_ref=None, + org_unit_types_ref=[self.org_unit_type_country], + # Options. + ignore_groups=True, + show_deleted_org_units=False, + field_names=["name"], + ) + + json_diffs = diffs_to_json(diffs) + + expected_json_diffs = [ + { + "org_unit": { + "id": self.angola_country_to_compare_with.pk, + "version": self.source_version_to_compare_with.pk, + "source_ref": "id-1", + "location": None, + "org_unit_type": self.org_unit_type_country.pk, + "path": str(self.angola_country_to_compare_with.path), + "name": "Angola new", + "parent": None, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "groups": [self.group_a2.pk, self.group_c.pk], + }, + "orgunit_ref": { + "id": self.angola_country_to_compare_with.pk, + "version": self.source_version_to_compare_with.pk, + "source_ref": "id-1", + "location": None, + "org_unit_type": self.org_unit_type_country.pk, + "path": str(self.angola_country_to_compare_with.path), + "name": "Angola new", + "parent": None, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "groups": [self.group_a2.pk, self.group_c.pk], + }, + "orgunit_dhis2": { + "id": self.angola_country_to_update.pk, + "version": self.source_version_to_update.pk, + "source_ref": "id-1", + "location": None, + "org_unit_type": self.org_unit_type_country.pk, + "path": str(self.angola_country_to_update.path), + "name": "Angola", + "parent": None, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "groups": [self.group_a1.pk, self.group_b.pk], + }, + "status": "modified", + "comparisons": [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + } + ], + } + ] + + self.assertJSONEqual(json_diffs, expected_json_diffs) + + def test_dump_as_json_for_org_unit_creation(self): + """ + Test the format of `Dumper.as_json()` for a new org unit. + """ + + # Simulate an org unit existing only in one pyramid. + self.angola_country_to_update.delete() + + # Limit the diff size with restrictions on `field_names` and `org_unit_types_ref`. + diffs, fields = Differ(test_logger).diff( + # Version to update. + version=self.source_version_to_update, + validation_status=None, + top_org_unit=None, + org_unit_types=[self.org_unit_type_country], + # Version to compare with. + version_ref=self.source_version_to_compare_with, + validation_status_ref=None, + top_org_unit_ref=None, + org_unit_types_ref=[self.org_unit_type_country], + # Options. + ignore_groups=True, + show_deleted_org_units=False, + field_names=["name"], + ) + + json_diffs = diffs_to_json(diffs) + + expected_json_diffs = [ + { + "org_unit": { + "id": self.angola_country_to_compare_with.pk, + "version": self.source_version_to_compare_with.pk, + "path": str(self.angola_country_to_compare_with.path), + "location": None, + "org_unit_type": self.org_unit_type_country.pk, + "source_ref": "id-1", + "name": "Angola", + "parent": None, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "groups": [self.group_a2.pk, self.group_c.pk], + }, + "orgunit_ref": { + "id": self.angola_country_to_compare_with.pk, + "version": self.source_version_to_compare_with.pk, + "source_ref": "id-1", + "path": str(self.angola_country_to_compare_with.path), + "location": None, + "org_unit_type": self.org_unit_type_country.pk, + "name": "Angola", + "parent": None, + "opening_date": "2022-11-28", + "closed_date": "2025-11-28", + "groups": [self.group_a2.pk, self.group_c.pk], + }, + "orgunit_dhis2": None, + "status": "new", + "comparisons": [ + { + "field": "name", + "before": None, + "after": "Angola", + "status": "new", + "distance": None, + } + ], + } + ] + + self.assertJSONEqual(json_diffs, expected_json_diffs) diff --git a/iaso/tests/diffing/utils.py b/iaso/tests/diffing/utils.py new file mode 100644 index 0000000000..71b281d846 --- /dev/null +++ b/iaso/tests/diffing/utils.py @@ -0,0 +1,121 @@ +import datetime + +from django.contrib.gis.geos import MultiPolygon, Polygon +from django.test import TestCase + +from iaso import models as m + + +class PyramidBaseTest(TestCase): + @classmethod + def setUpTestData(cls): + cls.data_source = m.DataSource.objects.create(name="Data source") + + cls.source_version_to_update = m.SourceVersion.objects.create( + data_source=cls.data_source, number=1, description="Bar" + ) + cls.source_version_to_compare_with = m.SourceVersion.objects.create( + data_source=cls.data_source, number=2, description="Foo" + ) + + cls.org_unit_type_country = m.OrgUnitType.objects.create(category="COUNTRY") + cls.org_unit_type_region = m.OrgUnitType.objects.create(category="REGION") + cls.org_unit_type_district = m.OrgUnitType.objects.create(category="DISTRICT") + + # Groups in the pyramid to update. + + cls.group_a1 = m.Group.objects.create( + name="Group A", source_ref="group-a", source_version=cls.source_version_to_update + ) + cls.group_b = m.Group.objects.create( + name="Group B", source_ref="group-b", source_version=cls.source_version_to_update + ) + + # Groups in the pyramid to compare with. + + cls.group_a2 = m.Group.objects.create( + name="Group A", source_ref="group-a", source_version=cls.source_version_to_compare_with + ) + cls.group_c = m.Group.objects.create( + name="Group C", source_ref="group-c", source_version=cls.source_version_to_compare_with + ) + + cls.multi_polygon = MultiPolygon(Polygon([(0, 0), (0, 1), (1, 1), (0, 0)])) + + # Angola pyramid to update. + + cls.angola_country_to_update = m.OrgUnit.objects.create( + parent=None, + version=cls.source_version_to_update, + source_ref="id-1", + name="Angola", + validation_status=m.OrgUnit.VALIDATION_VALID, + org_unit_type=cls.org_unit_type_country, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + ) + cls.angola_country_to_update.groups.set([cls.group_a1, cls.group_b]) + + cls.angola_region_to_update = m.OrgUnit.objects.create( + parent=cls.angola_country_to_update, + version=cls.source_version_to_update, + source_ref="id-2", + name="Huila", + org_unit_type=cls.org_unit_type_region, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + ) + + cls.angola_district_to_update = m.OrgUnit.objects.create( + parent=cls.angola_region_to_update, + version=cls.source_version_to_update, + source_ref="id-3", + name="Cuvango", + org_unit_type=cls.org_unit_type_district, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + ) + + # Angola pyramid to compare with. + + cls.angola_country_to_compare_with = m.OrgUnit.objects.create( + parent=None, + version=cls.source_version_to_compare_with, + source_ref="id-1", + name="Angola", + validation_status=m.OrgUnit.VALIDATION_VALID, + org_unit_type=cls.org_unit_type_country, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + ) + cls.angola_country_to_compare_with.groups.set([cls.group_a2, cls.group_c]) + + cls.angola_region_to_compare_with = m.OrgUnit.objects.create( + parent=cls.angola_country_to_compare_with, + version=cls.source_version_to_compare_with, + source_ref="id-2", + name="Huila", + org_unit_type=cls.org_unit_type_region, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + ) + + cls.angola_district_to_compare_with = m.OrgUnit.objects.create( + parent=cls.angola_region_to_compare_with, + version=cls.source_version_to_compare_with, + source_ref="id-3", + name="Cuvango", + org_unit_type=cls.org_unit_type_district, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + ) diff --git a/iaso/tests/gpkg/test_export2.py b/iaso/tests/gpkg/test_export2.py index 8e85059ce6..29429676f7 100644 --- a/iaso/tests/gpkg/test_export2.py +++ b/iaso/tests/gpkg/test_export2.py @@ -33,10 +33,10 @@ def setUpTestData(cls): name="ou2", version=cls.version, org_unit_type=out2, parent=ou, geom=polygon, simplified_geom=polygon ) m.OrgUnit.objects.create(name="ou3", version=cls.version, parent=ou2) # no orgunit type and no geom - group1 = m.Group.objects.create(name="group1", source_version=cls.version) - group2 = m.Group.objects.create(name="group2", source_ref="my_group_ref", source_version=cls.version) - ou.groups.add(group1) - ou2.groups.set([group1, group2]) + cls.group1 = m.Group.objects.create(name="group1", source_version=cls.version) + cls.group2 = m.Group.objects.create(name="group2", source_ref="my_group_ref", source_version=cls.version) + ou.groups.add(cls.group1) + ou2.groups.set([cls.group1, cls.group2]) def setUp(self): """Make sure we have a fresh client at the beginning of each test""" @@ -72,15 +72,23 @@ def test_export_import(self): self.assertEqual(root.name, "ou1") self.assertEqual(root.org_unit_type.name, "type1") self.assertEqual(root.orgunit_set.count(), 1) - self.assertQuerySetEqual(root.groups.all(), [""], transform=repr) + + self.assertEqual(root.groups.count(), 1) + first_group = root.groups.first() + self.assertEqual(first_group.name, self.group1.name) + self.assertEqual(first_group.source_version_id, v2.id) + c1 = root.orgunit_set.first() self.assertEqual(c1.name, "ou2") self.assertEqual(c1.org_unit_type.name, "type2") - self.assertQuerySetEqual( - c1.groups.all().order_by("name"), - ["", ""], - transform=repr, - ) + + c1_groups = c1.groups.all().order_by("name") + self.assertEqual(c1_groups.count(), 2) + self.assertEqual(c1_groups[0].name, self.group1.name) + self.assertEqual(c1_groups[0].source_version_id, v2.id) + self.assertEqual(c1_groups[1].name, self.group2.name) + self.assertEqual(c1_groups[1].source_version_id, v2.id) + self.assertEqual(c1.geom, c1.simplified_geom) self.assertEqual(c1.geom, self.polygon) c2 = c1.orgunit_set.first() diff --git a/iaso/tests/gpkg/test_import.py b/iaso/tests/gpkg/test_import.py index 6913b1ad38..c8e09f0513 100644 --- a/iaso/tests/gpkg/test_import.py +++ b/iaso/tests/gpkg/test_import.py @@ -65,11 +65,13 @@ def test_minimal_import(self): self.assertEqual(c2.geom, None) self.assertEqual(c2.simplified_geom, None) self.assertEqual(c2.location, Point(13.9993, 5.1795, 0.0, srid=4326)) - self.assertQuerySetEqual( - c2.groups.all().order_by("source_ref"), - ["", ""], - transform=repr, - ) + + c2_groups = c2.groups.all().order_by("source_ref") + self.assertEqual(c2_groups.count(), 2) + self.assertEqual(c2_groups[0].name, "Group A") + self.assertEqual(c2_groups[0].source_version.number, 1) + self.assertEqual(c2_groups[1].name, "Group B") + self.assertEqual(c2_groups[1].source_version.number, 1) self.assertEqual(OrgUnitType.objects.count(), 3) self.assertEqual(DataSource.objects.count(), 1) @@ -84,10 +86,14 @@ def test_minimal_import_modify_existing(self): ou = OrgUnit.objects.create(name="bla", source_ref="cdd3e94c-3c2a-4ab1-8900-be97f82347de", version=version) g = Group.objects.create(source_version=version, source_ref="group_b", name="Previous name of group B") ou.groups.set([g]) - self.assertQuerySetEqual(ou.groups.all(), [""], transform=repr) + self.assertEqual(ou.groups.count(), 1) + self.assertEqual(ou.groups.first().name, "Previous name of group B") + self.assertEqual(ou.groups.first().source_version, version) ou2 = OrgUnit.objects.create(name="bla2", source_ref="3c24c6ca-3012-4d38-abe8-6d620fe1deb8", version=version) ou2.groups.set([g]) - self.assertQuerySetEqual(ou2.groups.all(), [""], transform=repr) + self.assertEqual(ou2.groups.count(), 1) + self.assertEqual(ou2.groups.first().name, "Previous name of group B") + self.assertEqual(ou2.groups.first().source_version, version) import_gpkg_file( "./iaso/tests/fixtures/gpkg/minimal.gpkg", @@ -126,11 +132,14 @@ def test_minimal_import_modify_existing(self): self.assertEqual(g.name, "Group B") ou2.refresh_from_db() - self.assertQuerySetEqual( - ou2.groups.all().order_by("source_ref"), - ["", ""], - transform=repr, - ) + + ou2_groups = ou2.groups.all().order_by("source_ref") + self.assertEqual(ou2_groups.count(), 2) + self.assertEqual(ou2_groups[0].name, "Group A") + self.assertEqual(ou2_groups[0].source_version.number, version_number) + self.assertEqual(ou2_groups[1].name, "Group B") + self.assertEqual(ou2_groups[1].source_version.number, version_number) + mod = mods.get(object_id=ou2.id) old = mod.past_value[0] new = mod.new_value[0] diff --git a/iaso/tests/models/test_data_source_versions_synchronization.py b/iaso/tests/models/test_data_source_versions_synchronization.py new file mode 100644 index 0000000000..a7f23391d7 --- /dev/null +++ b/iaso/tests/models/test_data_source_versions_synchronization.py @@ -0,0 +1,463 @@ +import datetime +import json +import logging + +import time_machine + +from django.contrib.gis.geos import MultiPolygon, Polygon +from django.core.exceptions import ValidationError + +from iaso import models as m +from iaso.test import TestCase + + +class DataSourceVersionsSynchronizationModelTestCase(TestCase): + """ + Test DataSourceVersionsSynchronization model. + """ + + DT = datetime.datetime(2024, 12, 4, 17, 0, 0, 0, tzinfo=datetime.timezone.utc) + + def setUp(self): + logging.disable(logging.NOTSET) + + def tearDown(self): + logging.disable(logging.CRITICAL) + + @classmethod + def setUpTestData(cls): + cls.multi_polygon = MultiPolygon(Polygon([(0, 0), (0, 1), (1, 1), (0, 0)])) + + # Data source. + cls.data_source = m.DataSource.objects.create(name="Data source") + + # Data source versions. + cls.source_version_to_compare_with = m.SourceVersion.objects.create( + data_source=cls.data_source, number=1, description="Source version to compare with" + ) + cls.source_version_to_update = m.SourceVersion.objects.create( + data_source=cls.data_source, number=2, description="Source version to update" + ) + + # Groups in the pyramid to update. + + cls.group_a1 = m.Group.objects.create( + name="Group A", source_ref="group-a", source_version=cls.source_version_to_update + ) + cls.group_b = m.Group.objects.create( + name="Group B", source_ref="group-b", source_version=cls.source_version_to_update + ) + + # Groups in the pyramid to compare with. + + cls.group_a2 = m.Group.objects.create( + name="Group A", source_ref="group-a", source_version=cls.source_version_to_compare_with + ) + cls.group_c = m.Group.objects.create( + name="Group C", source_ref="group-c", source_version=cls.source_version_to_compare_with + ) + + # Org unit type. + cls.org_unit_type_country = m.OrgUnitType.objects.create(category="COUNTRY") + cls.org_unit_type_region = m.OrgUnitType.objects.create(category="REGION") + cls.org_unit_type_district = m.OrgUnitType.objects.create(category="DISTRICT") + cls.org_unit_type_facility = m.OrgUnitType.objects.create(category="FACILITY") + + # Angola pyramid to update (2 org units). + + cls.angola_country_to_update = m.OrgUnit.objects.create( + parent=None, + version=cls.source_version_to_update, + source_ref="id-1", + name="Angola", + validation_status=m.OrgUnit.VALIDATION_VALID, + org_unit_type=cls.org_unit_type_country, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + simplified_geom=cls.multi_polygon, + ) + cls.angola_country_to_update.groups.set([cls.group_a1, cls.group_b]) + + cls.angola_region_to_update = m.OrgUnit.objects.create( + parent=cls.angola_country_to_update, + version=cls.source_version_to_update, + source_ref="id-2", + name="Huila", + org_unit_type=cls.org_unit_type_region, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + simplified_geom=cls.multi_polygon, + ) + cls.angola_region_to_update.calculate_paths() + + # Angola pyramid to compare with (3 org units). + + cls.angola_country_to_compare_with = m.OrgUnit.objects.create( + parent=None, + version=cls.source_version_to_compare_with, + source_ref="id-1", + name="Angola", + validation_status=m.OrgUnit.VALIDATION_VALID, + org_unit_type=cls.org_unit_type_country, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + simplified_geom=cls.multi_polygon, + ) + cls.angola_country_to_compare_with.groups.set([cls.group_a2, cls.group_c]) + + cls.angola_region_to_compare_with = m.OrgUnit.objects.create( + parent=cls.angola_country_to_compare_with, + version=cls.source_version_to_compare_with, + source_ref="id-2", + name="Huila", + org_unit_type=cls.org_unit_type_region, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + simplified_geom=cls.multi_polygon, + ) + + cls.angola_district_to_compare_with = m.OrgUnit.objects.create( + parent=cls.angola_region_to_compare_with, + version=cls.source_version_to_compare_with, + source_ref="id-3", + name="Cuvango", + org_unit_type=cls.org_unit_type_district, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2022, 11, 28), + closed_date=datetime.date(2025, 11, 28), + geom=cls.multi_polygon, + simplified_geom=cls.multi_polygon, + ) + cls.angola_district_to_compare_with.groups.set([cls.group_a2, cls.group_c]) + + cls.angola_facility_to_compare_with = m.OrgUnit.objects.create( + parent=cls.angola_district_to_compare_with, + version=cls.source_version_to_compare_with, + source_ref="id-4", + name="Facility", + org_unit_type=cls.org_unit_type_facility, + validation_status=m.OrgUnit.VALIDATION_VALID, + opening_date=datetime.date(2024, 11, 28), + closed_date=datetime.date(2026, 11, 28), + geom=cls.multi_polygon, + simplified_geom=cls.multi_polygon, + ) + + cls.account = m.Account.objects.create(name="Account") + cls.user = cls.create_user_with_profile(username="user", account=cls.account) + + # Calculate paths. + cls.angola_country_to_update.calculate_paths() + cls.angola_country_to_compare_with.calculate_paths() + + @time_machine.travel(DT, tick=False) + def test_create(self): + kwargs = { + "name": "New synchronization", + "source_version_to_update": self.source_version_to_update, + "source_version_to_compare_with": self.source_version_to_compare_with, + "json_diff": None, + "account": self.account, + "created_by": self.user, + } + data_source_sync = m.DataSourceVersionsSynchronization(**kwargs) + data_source_sync.full_clean() + data_source_sync.save() + data_source_sync.refresh_from_db() + + self.assertEqual(data_source_sync.name, kwargs["name"]) + self.assertEqual(data_source_sync.source_version_to_update, kwargs["source_version_to_update"]) + self.assertEqual(data_source_sync.source_version_to_compare_with, kwargs["source_version_to_compare_with"]) + self.assertEqual(data_source_sync.json_diff, kwargs["json_diff"]) + self.assertIsNone(data_source_sync.sync_task) + self.assertEqual(data_source_sync.account, kwargs["account"]) + self.assertEqual(data_source_sync.created_by, kwargs["created_by"]) + self.assertEqual(data_source_sync.created_at, self.DT) + self.assertEqual(data_source_sync.updated_at, self.DT) + + def test_clean_data_source_versions(self): + other_data_source = m.DataSource.objects.create(name="Other data source") + other_source_version = m.SourceVersion.objects.create( + data_source=other_data_source, number=1, description="Other data source version" + ) + kwargs = { + "name": "Foo", + "source_version_to_update": self.source_version_to_update, + "source_version_to_compare_with": other_source_version, + "account": self.account, + "created_by": self.user, + } + data_source_sync = m.DataSourceVersionsSynchronization(**kwargs) + + with self.assertRaises(ValidationError) as error: + data_source_sync.clean_data_source_versions() + self.assertIn("The two versions to compare must be linked to the same data source.", error.exception.messages) + + kwargs = { + "name": "Foo", + "source_version_to_update": self.source_version_to_update, + "source_version_to_compare_with": self.source_version_to_update, + "account": self.account, + "created_by": self.user, + } + data_source_sync = m.DataSourceVersionsSynchronization(**kwargs) + + with self.assertRaises(ValidationError) as error: + data_source_sync.clean_data_source_versions() + self.assertIn("The two versions to compare must be different.", error.exception.messages) + + def test_create_json_diff(self): + """ + Test that `create_json_diff()` works as expected. + """ + # Change the name. + self.angola_country_to_compare_with.name = "Angola new" + self.angola_country_to_compare_with.save() + + data_source_sync = m.DataSourceVersionsSynchronization.objects.create( + name="New synchronization", + source_version_to_update=self.source_version_to_update, + source_version_to_compare_with=self.source_version_to_compare_with, + json_diff=None, + account=self.account, + created_by=self.user, + ) + self.assertIsNone(data_source_sync.json_diff) + self.assertEqual(data_source_sync.diff_config, "") + + data_source_sync.create_json_diff( + source_version_to_update_validation_status=m.OrgUnit.VALIDATION_VALID, + source_version_to_update_org_unit_types=[self.org_unit_type_country], + source_version_to_compare_with_validation_status=m.OrgUnit.VALIDATION_VALID, + source_version_to_compare_with_org_unit_types=[self.org_unit_type_country], + ignore_groups=True, + field_names=["name"], + ) + data_source_sync.refresh_from_db() + + json_diff = json.loads(data_source_sync.json_diff) + + comparisons = json_diff[0]["comparisons"] + expected_comparisons = [ + { + "field": "name", + "before": "Angola", + "after": "Angola new", + "status": "modified", + "distance": None, + } + ] + self.assertEqual(comparisons, expected_comparisons) + + expected_diff_config = ( + "{" + f"'version': {self.source_version_to_update.pk}, " + f"'validation_status': '{m.OrgUnit.VALIDATION_VALID}', " + "'top_org_unit': None, " + f"'org_unit_types': [{self.org_unit_type_country.pk}], " + f"'version_ref': {self.source_version_to_compare_with.pk}, " + f"'validation_status_ref': '{m.OrgUnit.VALIDATION_VALID}', " + "'top_org_unit_ref': None, " + f"'org_unit_types_ref': [{self.org_unit_type_country.pk}], " + "'ignore_groups': True, " + "'show_deleted_org_units': False, " + "'field_names': ['name']" + "}" + ) + self.assertEqual(data_source_sync.diff_config, expected_diff_config) + + self.assertEqual(data_source_sync.count_create, 0) + self.assertEqual(data_source_sync.count_update, 1) + + def test_create_change_requests(self): + """ + Test that `create_change_requests()` produces 3 change requests: + - 2 change requests to modify existing org units + - 1 change request to create a new org unit + """ + # Changes at the country level. + self.angola_country_to_compare_with.name = "Angola new" + self.angola_country_to_compare_with.opening_date = datetime.date(2025, 11, 28) + self.angola_country_to_compare_with.closed_date = datetime.date(2026, 11, 28) + self.angola_country_to_compare_with.save() + # Changes at the region level. + self.angola_region_to_compare_with.parent = None + self.angola_region_to_compare_with.opening_date = datetime.date(2025, 11, 28) + self.angola_region_to_compare_with.closed_date = datetime.date(2026, 11, 28) + self.angola_region_to_compare_with.save() + + data_source_sync = m.DataSourceVersionsSynchronization.objects.create( + name="New synchronization", + source_version_to_update=self.source_version_to_update, + source_version_to_compare_with=self.source_version_to_compare_with, + json_diff=None, + account=self.account, + created_by=self.user, + ) + data_source_sync.create_json_diff() + + # Synchronize source versions. + change_requests = m.OrgUnitChangeRequest.objects.filter(data_source_synchronization=data_source_sync) + self.assertEqual(change_requests.count(), 0) + self.assertEqual(m.OrgUnit.objects.filter(version=self.source_version_to_update).count(), 2) + + with self.assertNumQueries(12): + data_source_sync.synchronize_source_versions() + + self.assertEqual(change_requests.count(), 4) + self.assertEqual(m.OrgUnit.objects.filter(version=self.source_version_to_update).count(), 4) + + # Change request #1 to update an existing OrgUnit. + angola_country_change_request = m.OrgUnitChangeRequest.objects.get( + org_unit=self.angola_country_to_update, data_source_synchronization=data_source_sync + ) + # Data. + self.assertEqual(angola_country_change_request.kind, m.OrgUnitChangeRequest.Kind.ORG_UNIT_CHANGE) + self.assertEqual(angola_country_change_request.created_by, data_source_sync.created_by) + self.assertEqual( + angola_country_change_request.requested_fields, + ["new_name", "new_opening_date", "new_closed_date", "new_groups"], + ) + # New values. + self.assertEqual(angola_country_change_request.new_parent, None) + self.assertEqual(angola_country_change_request.new_name, "Angola new") + self.assertEqual(angola_country_change_request.new_org_unit_type, None) + self.assertEqual(angola_country_change_request.new_groups.count(), 2) + self.assertIn(self.group_a2, angola_country_change_request.new_groups.all()) + self.assertIn(self.group_c, angola_country_change_request.new_groups.all()) + self.assertEqual(angola_country_change_request.new_location, None) + self.assertEqual(angola_country_change_request.new_location_accuracy, None) + self.assertEqual(angola_country_change_request.new_opening_date, datetime.date(2025, 11, 28)) + self.assertEqual(angola_country_change_request.new_closed_date, datetime.date(2026, 11, 28)) + self.assertEqual(angola_country_change_request.new_reference_instances.count(), 0) + # Old values. + self.assertEqual(angola_country_change_request.old_parent, None) + self.assertEqual(angola_country_change_request.old_name, "Angola") + self.assertEqual(angola_country_change_request.old_org_unit_type, self.org_unit_type_country) + self.assertEqual(angola_country_change_request.old_groups.count(), 2) + self.assertIn(self.group_a1, angola_country_change_request.old_groups.all()) + self.assertIn(self.group_b, angola_country_change_request.old_groups.all()) + self.assertEqual(angola_country_change_request.old_location, None) + self.assertEqual(angola_country_change_request.old_opening_date, datetime.date(2022, 11, 28)) + self.assertEqual(angola_country_change_request.old_closed_date, datetime.date(2025, 11, 28)) + self.assertEqual(angola_country_change_request.old_reference_instances.count(), 0) + + # Change request #2 to update an existing OrgUnit. + angola_region_change_request = m.OrgUnitChangeRequest.objects.get( + org_unit=self.angola_region_to_update, data_source_synchronization=data_source_sync + ) + # Data. + self.assertEqual(angola_region_change_request.kind, m.OrgUnitChangeRequest.Kind.ORG_UNIT_CHANGE) + self.assertEqual(angola_region_change_request.created_by, data_source_sync.created_by) + self.assertEqual(angola_region_change_request.requested_fields, ["new_opening_date", "new_closed_date"]) + # New values. + self.assertEqual(angola_region_change_request.new_parent, None) + self.assertEqual(angola_region_change_request.new_name, "") + self.assertEqual(angola_region_change_request.new_org_unit_type, None) + self.assertEqual(angola_region_change_request.new_groups.count(), 0) + self.assertEqual(angola_region_change_request.new_location, None) + self.assertEqual(angola_region_change_request.new_location_accuracy, None) + self.assertEqual(angola_region_change_request.new_opening_date, datetime.date(2025, 11, 28)) + self.assertEqual(angola_region_change_request.new_closed_date, datetime.date(2026, 11, 28)) + self.assertEqual(angola_region_change_request.new_reference_instances.count(), 0) + # Old values. + self.assertEqual(angola_region_change_request.old_parent, self.angola_country_to_update) + self.assertEqual(angola_region_change_request.old_name, "Huila") + self.assertEqual(angola_region_change_request.old_org_unit_type, self.org_unit_type_region) + self.assertEqual(angola_region_change_request.old_groups.count(), 0) + self.assertEqual(angola_region_change_request.old_location, None) + self.assertEqual(angola_region_change_request.old_opening_date, datetime.date(2022, 11, 28)) + self.assertEqual(angola_region_change_request.old_closed_date, datetime.date(2025, 11, 28)) + self.assertEqual(angola_region_change_request.old_reference_instances.count(), 0) + + # Change request #3 to create a new OrgUnit whose corresponding parent DID exist. + new_group_c = m.Group.objects.get(name="Group C", source_version=data_source_sync.source_version_to_update) + new_group_a2 = m.Group.objects.get(name="Group A", source_version=data_source_sync.source_version_to_update) + angola_district_change_request = m.OrgUnitChangeRequest.objects.get( + org_unit__source_ref="id-3", data_source_synchronization=data_source_sync + ) + # Data. + self.assertEqual(angola_district_change_request.kind, m.OrgUnitChangeRequest.Kind.ORG_UNIT_CREATION) + self.assertEqual(angola_district_change_request.created_by, data_source_sync.created_by) + self.assertEqual( + angola_district_change_request.requested_fields, + ["new_name", "new_parent", "new_opening_date", "new_closed_date", "new_groups"], + ) + # New values. + self.assertEqual(angola_district_change_request.new_parent, self.angola_region_to_update) + self.assertEqual(angola_district_change_request.new_name, "Cuvango") + self.assertEqual(angola_district_change_request.new_org_unit_type, None) + self.assertEqual(angola_district_change_request.new_groups.count(), 2) + self.assertIn(new_group_c, angola_district_change_request.new_groups.all()) + self.assertIn(new_group_a2, angola_district_change_request.new_groups.all()) + self.assertEqual(angola_district_change_request.new_location, None) + self.assertEqual(angola_district_change_request.new_location_accuracy, None) + self.assertEqual(angola_district_change_request.new_opening_date, datetime.date(2022, 11, 28)) + self.assertEqual(angola_district_change_request.new_closed_date, datetime.date(2025, 11, 28)) + self.assertEqual(angola_district_change_request.new_reference_instances.count(), 0) + # Old values. + self.assertEqual(angola_district_change_request.old_parent, None) + self.assertEqual(angola_district_change_request.old_name, "") + self.assertEqual(angola_district_change_request.old_org_unit_type, None) + self.assertEqual(angola_district_change_request.old_groups.count(), 0) + self.assertEqual(angola_district_change_request.old_location, None) + self.assertEqual(angola_district_change_request.old_opening_date, None) + self.assertEqual(angola_district_change_request.old_closed_date, None) + self.assertEqual(angola_district_change_request.old_reference_instances.count(), 0) + + new_angola_district_org_unit = m.OrgUnit.objects.get(pk=angola_district_change_request.org_unit.pk) + self.assertEqual(new_angola_district_org_unit.version, data_source_sync.source_version_to_update) + self.assertEqual(new_angola_district_org_unit.parent.version, data_source_sync.source_version_to_update) + self.assertEqual(new_angola_district_org_unit.creator, data_source_sync.created_by) + self.assertEqual(new_angola_district_org_unit.validation_status, new_angola_district_org_unit.VALIDATION_NEW) + self.assertEqual(new_angola_district_org_unit.org_unit_type, self.org_unit_type_district) + self.assertEqual(new_angola_district_org_unit.source_ref, "id-3") + self.assertEqual(new_angola_district_org_unit.geom, self.multi_polygon) + self.assertEqual(new_angola_district_org_unit.simplified_geom, self.multi_polygon) + + # Change request #4 to create a new OrgUnit whose corresponding parent DID NOT exist. + angola_facility_change_request = m.OrgUnitChangeRequest.objects.get( + org_unit__source_ref="id-4", data_source_synchronization=data_source_sync + ) + # Data. + self.assertEqual(angola_facility_change_request.kind, m.OrgUnitChangeRequest.Kind.ORG_UNIT_CREATION) + self.assertEqual(angola_facility_change_request.created_by, data_source_sync.created_by) + self.assertEqual( + angola_facility_change_request.requested_fields, + ["new_name", "new_parent", "new_opening_date", "new_closed_date"], + ) + # New values. + self.assertEqual(angola_facility_change_request.new_parent, new_angola_district_org_unit) + self.assertEqual(angola_facility_change_request.new_name, "Facility") + self.assertEqual(angola_facility_change_request.new_org_unit_type, None) + self.assertEqual(angola_facility_change_request.new_groups.count(), 0) + self.assertEqual(angola_facility_change_request.new_location, None) + self.assertEqual(angola_facility_change_request.new_location_accuracy, None) + self.assertEqual(angola_facility_change_request.new_opening_date, datetime.date(2024, 11, 28)) + self.assertEqual(angola_facility_change_request.new_closed_date, datetime.date(2026, 11, 28)) + self.assertEqual(angola_facility_change_request.new_reference_instances.count(), 0) + # Old values. + self.assertEqual(angola_facility_change_request.old_parent, None) + self.assertEqual(angola_facility_change_request.old_name, "") + self.assertEqual(angola_facility_change_request.old_org_unit_type, None) + self.assertEqual(angola_facility_change_request.old_groups.count(), 0) + self.assertEqual(angola_facility_change_request.old_location, None) + self.assertEqual(angola_facility_change_request.old_opening_date, None) + self.assertEqual(angola_facility_change_request.old_closed_date, None) + self.assertEqual(angola_facility_change_request.old_reference_instances.count(), 0) + + new_angola_facility_org_unit = m.OrgUnit.objects.get(pk=angola_facility_change_request.org_unit.pk) + self.assertEqual(new_angola_facility_org_unit.version, data_source_sync.source_version_to_update) + self.assertEqual(new_angola_facility_org_unit.parent.version, data_source_sync.source_version_to_update) + self.assertEqual(new_angola_facility_org_unit.creator, data_source_sync.created_by) + self.assertEqual(new_angola_facility_org_unit.validation_status, new_angola_facility_org_unit.VALIDATION_NEW) + self.assertEqual(new_angola_facility_org_unit.org_unit_type, self.org_unit_type_facility) + self.assertEqual(new_angola_facility_org_unit.source_ref, "id-4") + self.assertEqual(new_angola_facility_org_unit.geom, self.multi_polygon) + self.assertEqual(new_angola_facility_org_unit.simplified_geom, self.multi_polygon) diff --git a/iaso/tests/test_create_users_from_csv.py b/iaso/tests/test_create_users_from_csv.py index 9fa4a348de..2d9e41e1e0 100644 --- a/iaso/tests/test_create_users_from_csv.py +++ b/iaso/tests/test_create_users_from_csv.py @@ -355,7 +355,7 @@ def test_cant_create_user_without_ou_profile(self): self.assertEqual( response.data, { - "error": "Operation aborted. Invalid OrgUnit None chiloe 10244 at row : 2. You don't have access to this orgunit" + "error": "Operation aborted. Invalid OrgUnit #10244 chiloe at row : 2. You don't have access to this orgunit" }, )