IA-3728 Create change requests from differences between two pyramids (backend model)

iaso/admin.py

@@ -27,6 +27,7 @@
     AlgorithmRun,
     BulkCreateUserCsvFile,
     DataSource,
+    DataSourceVersionsSynchronization,
     Device,
     DeviceOwnership,
     DevicePosition,
@@ -508,6 +509,7 @@ class TaskAdmin(admin.ModelAdmin):
     list_filter = ("account", "status", "name")
     readonly_fields = ("stacktrace", "created_at", "result")
     formfield_overrides = {models.JSONField: {"widget": IasoJSONEditorWidget}}
+    search_fields = ("name",)
 
     def result_message(self, task):
         return task.result and task.result.get("message", "")
@@ -526,7 +528,7 @@ def get_queryset(self, request):
 @admin_attr_decorator
 class SourceVersionAdmin(admin.ModelAdmin):
     readonly_fields = ("created_at",)
-    list_display = ["__str__", "data_source", "number", "created_at", "updated_at"]
+    list_display = ["id", "data_source", "number", "created_at", "updated_at"]
     list_filter = ["data_source", "created_at", "updated_at"]
     search_fields = ["data_source__name", "number", "description"]
     autocomplete_fields = ["data_source"]
@@ -824,7 +826,7 @@ class EntityDuplicateAnalyzisAdmin(admin.ModelAdmin):
 class OrgUnitChangeRequestAdmin(admin.ModelAdmin):
     list_display = ("pk", "org_unit", "created_at", "status")
     list_display_links = ("pk", "org_unit")
-    list_filter = ("status", "kind")
+    list_filter = ("status", "kind", "data_source_synchronization")
     readonly_fields = (
         "uuid",
         "created_at",
@@ -849,6 +851,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin):
         "new_reference_instances",
         "payment",
         "potential_payment",
+        "data_source_synchronization",
     )
     fieldsets = (
         (
@@ -898,6 +901,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin):
                     "updated_at",
                     "updated_by",
                     "rejection_comment",
+                    "data_source_synchronization",
                 )
             },
         ),
@@ -919,7 +923,7 @@ class OrgUnitChangeRequestAdmin(admin.ModelAdmin):
     )
 
     def get_queryset(self, request):
-        return super().get_queryset(request).select_related("org_unit__org_unit_type")
+        return super().get_queryset(request).select_related("org_unit__org_unit_type", "data_source_synchronization")
 
 
 @admin.register(Config)
@@ -1077,6 +1081,40 @@ class Media:
         }
 
 
+@admin.register(DataSourceVersionsSynchronization)
+class DataSourceVersionsSynchronizationAdmin(admin.ModelAdmin):
+    list_display = (
+        "pk",
+        "name",
+        "account",
+        "created_by",
+        "count_create",
+        "count_update",
+    )
+    list_display_links = ("pk", "name")
+    autocomplete_fields = ("account", "created_by", "source_version_to_update", "source_version_to_compare_with")
+    readonly_fields = (
+        "json_diff",
+        "count_create",
+        "count_update",
+        "created_at",
+        "updated_at",
+        "sync_task",
+    )
+
+    def get_queryset(self, request):
+        return (
+            super()
+            .get_queryset(request)
+            .select_related(
+                "source_version_to_update__data_source",
+                "source_version_to_compare_with__data_source",
+                "account",
+                "created_by",
+            )
+        )
+
+
 admin.site.register(AccountFeatureFlag)
 admin.site.register(Device)
 admin.site.register(DeviceOwnership)

iaso/diffing/__init__.py

@@ -2,3 +2,4 @@
 from .differ import Differ
 from .dumper import Dumper
 from .exporter import Exporter
+from .synchronizer import DataSourceVersionsSynchronizer, diffs_to_json

iaso/diffing/comparisons.py

@@ -92,7 +92,7 @@ def access(self, org_unit):
         groups = []
         for group in org_unit.groups.all():
             if group.source_ref == self.group_ref:
-                groups.append({"id": group.source_ref, "name": group.name})
+                groups.append({"id": group.source_ref, "name": group.name, "iaso_id": group.pk})
 
         return groups
 

iaso/diffing/differ.py

@@ -1,3 +1,5 @@
+from django.db.models import Q
+
 from iaso.models import OrgUnit, GroupSet, Group
 from .comparisons import as_field_types, Diff, Comparison
 
@@ -19,7 +21,7 @@ def __init__(self, logger):
         self.iaso_logger = logger
 
     def load_pyramid(self, version, validation_status=None, top_org_unit=None, org_unit_types=None):
-        self.iaso_logger.info("loading pyramid ", version.data_source, version, top_org_unit, org_unit_types)
+        self.iaso_logger.info(f"loading pyramid {version.data_source} {version} {top_org_unit} {org_unit_types}")
         queryset = (
             OrgUnit.objects.prefetch_related("groups")
             .prefetch_related("groups__group_sets")
@@ -62,11 +64,13 @@ def diff(
                     field_names.append("groupset:" + group_set.source_ref + ":" + group_set.name)
                     for group in group_set.groups.all():
                         groups_with_with_groupset.append(group.id)
-            for group in Group.objects.filter(source_version=version):
+            for group in Group.objects.filter(Q(source_version=version) | Q(source_version=version_ref)).distinct(
+                "source_ref"
+            ):
                 if group.id not in groups_with_with_groupset and group.source_ref:
                     field_names.append("group:" + group.source_ref + ":" + group.name)
 
-        self.iaso_logger.info("will compare the following fields ", field_names)
+        self.iaso_logger.info(f"will compare the following fields {field_names}")
         field_types = as_field_types(field_names)
 
         orgunits_dhis2 = self.load_pyramid(
@@ -78,9 +82,7 @@ def diff(
             top_org_unit=top_org_unit_ref,
             org_unit_types=org_unit_types_ref,
         )
-        self.iaso_logger.info(
-            "comparing ", version_ref, "(", len(orgunits_dhis2), ")", " and ", version, "(", len(orgunit_refs), ")"
-        )
+        self.iaso_logger.info(f"comparing {version_ref} ({len(orgunits_dhis2)}) and {version} ({len(orgunit_refs)})")
         # speed how to index_by(&:source_ref)
         diffs = []
         index = 0
@@ -146,7 +148,7 @@ def compare_fields(self, orgunit_dhis2, orgunit_ref, field_types):
             else:
                 status = "modified"
 
-            if dhis2_value is None and ref_value is not None:
+            if not dhis2_value and ref_value:
                 status = "new"
             if not same and dhis2_value is not None and (ref_value is None or ref_value == []):
                 status = "deleted"

iaso/diffing/dumper.py

@@ -2,6 +2,8 @@
 import json
 
 from django.contrib.gis.geos import GEOSGeometry
+from django.core.serializers.json import DjangoJSONEncoder
+from django.forms import model_to_dict
 
 from iaso.management.commands.command_logger import CommandLogger
 
@@ -18,14 +20,19 @@ def color(status):
     return CommandLogger.END
 
 
-class ShapelyJsonEncoder(json.JSONEncoder):
-    def __init__(self, **kwargs):
-        super(ShapelyJsonEncoder, self).__init__(**kwargs)
-
+class DiffJSONEncoder(DjangoJSONEncoder):
     def default(self, obj):
-        if hasattr(obj, "as_dict"):
+        if obj.__class__.__name__ in ["Diff", "Comparison"]:
             return obj.as_dict()
-        return obj.wkt
+        if obj.__class__.__name__ == "OrgUnit":
+            return model_to_dict(obj)
+        if obj.__class__.__name__ == "PathValue":
+            # See django_ltree.fields
+            # https://github.com/mariocesar/django-ltree/blob/154c7e/django_ltree/fields.py#L27-L28
+            return str(obj)
+        if obj.__class__.__name__ == "MultiPolygon":
+            return obj.wkt
+        return super().default(obj)
 
 
 class Dumper:
@@ -60,8 +67,11 @@ def dump_stats(self, diffs):
         self.iaso_logger.info(json.dumps(stats, indent=4))
         return stats
 
+    def as_json(self, diffs):
+        return json.dumps(diffs, indent=4, cls=DiffJSONEncoder)
+
     def dump_as_json(self, diffs):
-        self.iaso_logger.info(json.dumps(diffs, indent=4, cls=ShapelyJsonEncoder))
+        self.iaso_logger.info(self.as_json(diffs))
 
     def dump_as_csv(self, diffs, fields, csv_file, number_of_parents=5):
         res = []