From 57b30fc91bc0e6ed5fffd837f3a88a2cba908343 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 13 Dec 2024 19:05:50 +0100 Subject: [PATCH 01/62] API call check push gps - WIP --- iaso/api/instances.py | 109 +++++++++++++++++++- iaso/tests/api/test_instances.py | 170 +++++++++++++++++++++++++++++-- 2 files changed, 271 insertions(+), 8 deletions(-) diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 8e1b09f548..8f87b5a8ea 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -3,7 +3,7 @@ import ntpath from copy import copy from time import gmtime, strftime -from typing import Any, Dict, Union +from typing import Any, Dict, Union, List import pandas as pd from django.contrib.auth.models import User @@ -43,7 +43,14 @@ from ..utils.models.common import get_creator_name from . import common from .comment import UserSerializerForComment -from .common import CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, TimestampField, safe_api_import +from .common import ( + CONTENT_TYPE_CSV, + CONTENT_TYPE_XLSX, + FileFormatEnum, + TimestampField, + safe_api_import, + parse_comma_separated_numeric_values, +) from .instance_filters import get_form_from_instance_filters, parse_instance_filters logger = logging.getLogger(__name__) @@ -598,6 +605,104 @@ def bulkdelete(self, request): status=201, ) + @action(detail=False, methods=["GET"], permission_classes=[permissions.IsAuthenticated, HasInstancePermission]) + def check_bulk_gps_push(self, request): + # first, let's parse all parameters received from the URL + select_all, selected_ids, unselected_ids = self._parse_check_bulk_gps_push_parameters(request.GET) + + # then, let's make sure that each ID actually exists and that the user has access to it + instances_query = self.get_queryset() + for selected_id in selected_ids: + get_object_or_404(instances_query, pk=selected_id) + for unselected_id in unselected_ids: + get_object_or_404(instances_query, pk=unselected_id) + + # let's filter everything + filters = parse_instance_filters(request.GET) + instances_query = instances_query.select_related("org_unit") + instances_query = instances_query.exclude(file="").exclude(device__test_device=True) + instances_query = instances_query.for_filters(**filters) + + if not select_all: + instances_query = instances_query.filter(pk__in=selected_ids) + else: + instances_query = instances_query.exclude(pk__in=unselected_ids) + + overwrite_ids = [] + no_location_ids = [] + org_units_to_instances_dict = {} + set_org_units_ids = set() + + for instance in instances_query: + if not instance.location: + no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit + continue + + org_unit = instance.org_unit + if org_unit.id in org_units_to_instances_dict: + # we can't push this instance's location since there was another instance linked to this OrgUnit + org_units_to_instances_dict[org_unit.id].append(instance.id) + continue + else: + org_units_to_instances_dict[org_unit.id] = [instance.id] + + set_org_units_ids.add(org_unit.id) + if org_unit.location or org_unit.geom: + overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location + continue + + # Before returning, we need to check if we've had multiple hits on an OrgUnit + error_same_org_unit_ids = self._check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + + if len(error_same_org_unit_ids): + return Response( + {"result": "error", "error_ids": error_same_org_unit_ids}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if len(no_location_ids) or len(overwrite_ids): + dict_response = { + "result": "warnings", + } + if len(no_location_ids): + dict_response["warning_no_location"] = no_location_ids + if len(overwrite_ids): + dict_response["warning_overwrite"] = overwrite_ids + + return Response(dict_response, status=status.HTTP_200_OK) + + return Response( + { + "result": "success", + }, + status=status.HTTP_200_OK, + ) + + def _parse_check_bulk_gps_push_parameters(self, query_parameters): + raw_select_all = query_parameters.get("select_all", True) + select_all = raw_select_all not in ["false", "False", "0"] + + raw_selected_ids = query_parameters.get("selected_ids", None) + if raw_selected_ids: + selected_ids = parse_comma_separated_numeric_values(raw_selected_ids, "selected_ids") + else: + selected_ids = [] + + raw_unselected_ids = query_parameters.get("unselected_ids", None) + if raw_unselected_ids: + unselected_ids = parse_comma_separated_numeric_values(raw_unselected_ids, "unselected_ids") + else: + unselected_ids = [] + + return select_all, selected_ids, unselected_ids + + def _check_bulk_gps_repeated_org_units(self, org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: + error_instance_ids = [] + for _, instance_ids in org_units_to_instance_ids.items(): + if len(instance_ids) >= 2: + error_instance_ids.extend(instance_ids) + return error_instance_ids + QUERY = """ select DATE_TRUNC('month', COALESCE(iaso_instance.source_created_at, iaso_instance.created_at)) as month, (select name from iaso_form where id = iaso_instance.form_id) as form_name, diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 5675da1859..7bea3775a7 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -1925,16 +1925,174 @@ def test_instance_retrieve_with_related_change_requests(self): # Test instance with no change request response = self.client.get(f"/api/instances/{instance_reference.id}/") self.assertEqual(response.status_code, 200) - respons_json = response.json() - self.assertListEqual(respons_json["change_requests"], []) + response_json = response.json() + self.assertListEqual(response_json["change_requests"], []) m.OrgUnitChangeRequest.objects.create(org_unit=org_unit, new_name="Modified org unit") response = self.client.get(f"/api/instances/{instance_reference.id}/") - respons_json = response.json() - self.assertListEqual(respons_json["change_requests"], []) + response_json = response.json() + self.assertListEqual(response_json["change_requests"], []) # Test instance with change request linked to it change_request_instance_reference = m.OrgUnitChangeRequest.objects.create(org_unit=org_unit) change_request_instance_reference.new_reference_instances.add(instance_reference) response = self.client.get(f"/api/instances/{instance_reference.id}/") - respons_json = response.json() - self.assertEqual(respons_json["change_requests"][0]["id"], change_request_instance_reference.id) + response_json = response.json() + self.assertEqual(response_json["change_requests"][0]["id"], change_request_instance_reference.id) + + def test_check_bulk_push_gps_select_all_ok(self): + # pushing gps data means that we need a mapping of 1 instance to 1 orgunit + for instance in [self.instance_2, self.instance_3, self.instance_4]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + instance.refresh_from_db() + + # setting gps data for instances that were not deleted + self.instance_5.org_unit = self.jedi_council_endor + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + new_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_5, self.instance_6, self.instance_8]: + instance.location = new_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + self.assertEqual(response_json["result"], "success") + + def test_check_bulk_push_gps_select_all_error(self): + # setting gps data for instances that were not deleted + self.instance_1.org_unit = self.jedi_council_endor + self.instance_2.org_unit = self.jedi_council_endor + new_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_2]: + instance.location = new_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "error") + self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + + def test_check_bulk_push_gps_select_all_warning_no_location(self): + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # Since all instances in the setup don't have location data, they will show up in warnings + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual( + response_json["warning_no_location"], m.Instance.non_deleted_objects.values_list("id", flat=True) + ) + + def test_check_bulk_push_gps_select_all_warning_overwrite(self): + # pushing gps data means that we need a mapping of 1 instance to 1 orgunit + for instance in [self.instance_2, self.instance_3, self.instance_4]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + instance.refresh_from_db() + + # no org unit was assigned, so we select some + self.instance_5.org_unit = self.jedi_council_endor + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + new_org_unit_location = Point(1, 2, 3) + new_instance_location = Point(4, 5, 6) + non_deleted_instances = [self.instance_1, self.instance_5, self.instance_6, self.instance_8] + for instance in non_deleted_instances: + # setting gps data for org_units whose instance was not deleted + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + # setting gps data for these instances as well, since we want to override the org_unit location + instance.location = new_instance_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [i.id for i in non_deleted_instances]) + + def test_check_bulk_push_gps_selected_ids_ok(self): + self.client.force_authenticate(self.yoda) + new_instance = self.create_form_instance( + form=self.form_1, + period="2024Q4", + org_unit=self.jedi_council_corruscant, + project=self.project, + created_by=self.yoda, + location=Point(5, 6.45, 2.33), + ) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={new_instance.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # Instance has a location, but OrgUnit doesn't, so check is ok, location could be pushed + self.assertEqual(response_json["result"], "success") + + def test_check_bulk_push_gps_selected_ids_error(self): + self.client.force_authenticate(self.yoda) + # Setting GPS data for these 3 instances (same OrgUnit) + for instance in [self.instance_1, self.instance_2, self.instance_3]: + instance.location = Point(5, 6.45, 2.33) + instance.save() + self.instance_1.refresh_from_db() + self.instance_2.refresh_from_db() + self.instance_3.refresh_from_db() + + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id},{self.instance_3.id}" + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "error") + self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + + def test_check_bulk_push_gps_selected_ids_warning_no_location(self): + self.client.force_authenticate(self.yoda) + # Linking these instances to some orgunits + self.instance_1.org_unit = self.jedi_council_corruscant + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + for instance in [self.instance_1, self.instance_2, self.instance_3]: + instance.save() + instance.refresh_from_db() + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id},{self.instance_3.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual( + response_json["warning_no_location"], [self.instance_1.id, self.instance_2.id, self.instance_3.id] + ) + + def test_check_bulk_push_gps_selected_ids_warning_overwrite(self): + pass + + def test_check_bulk_push_gps_unselected_ids_ok(self): + pass + + def test_check_bulk_push_gps_unselected_ids_error(self): + pass + + def test_check_bulk_push_gps_unselected_ids_warning_no_location(self): + pass + + def test_check_bulk_push_gps_unselected_ids_warning_overwrite(self): + pass From 7fa697574caef83ec1c694aa0f09eafa6f8a3740 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 13 Dec 2024 20:34:49 +0100 Subject: [PATCH 02/62] Added tests with wrong account and wrong IDs --- iaso/tests/api/test_instances.py | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 7bea3775a7..7ac3e697f1 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -2062,6 +2062,44 @@ def test_check_bulk_push_gps_selected_ids_error(self): self.assertEqual(response_json["result"], "error") self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): + self.client.force_authenticate(self.yoda) + probably_not_a_valid_id = 1234567980 + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={probably_not_a_valid_id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + + def test_check_bulk_push_gps_selected_ids_error_wrong_account(self): + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) + new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + new_instance = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + ) + + self.client.force_authenticate(self.yoda) + # Checking instance from new account + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={new_instance.id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + def test_check_bulk_push_gps_selected_ids_warning_no_location(self): self.client.force_authenticate(self.yoda) # Linking these instances to some orgunits @@ -2091,6 +2129,44 @@ def test_check_bulk_push_gps_unselected_ids_ok(self): def test_check_bulk_push_gps_unselected_ids_error(self): pass + def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): + self.client.force_authenticate(self.yoda) + probably_not_a_valid_id = 1234567980 + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&unselected_ids={probably_not_a_valid_id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + + def test_check_bulk_push_gps_unselected_ids_error_wrong_account(self): + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) + new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + new_instance = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + ) + + self.client.force_authenticate(self.yoda) + # Checking instance from new account + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&unselected_ids={new_instance.id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + def test_check_bulk_push_gps_unselected_ids_warning_no_location(self): pass From 84d43de18074f1c919ae0d49474b49eef7f7f004 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Sat, 14 Dec 2024 00:20:40 +0100 Subject: [PATCH 03/62] Added missing tests --- iaso/tests/api/test_instances.py | 232 +++++++++++++++++++++++++++++-- 1 file changed, 220 insertions(+), 12 deletions(-) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 7ac3e697f1..e7defdfddc 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -2025,6 +2025,44 @@ def test_check_bulk_push_gps_select_all_warning_overwrite(self): self.assertEqual(response_json["result"], "warnings") self.assertCountEqual(response_json["warning_overwrite"], [i.id for i in non_deleted_instances]) + def test_check_bulk_push_gps_select_all_warning_both(self): + # pushing gps data means that we need a mapping of 1 instance to 1 orgunit + for instance in [self.instance_2, self.instance_3, self.instance_4]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + instance.refresh_from_db() + + # instances 1 and 5 will overwrite their org_unit location + new_instance_location = Point(4, 5, 6) + self.instance_5.org_unit = self.jedi_council_endor + self.instance_5.location = new_instance_location + self.instance_1.location = new_instance_location + + # instances 6 and 8 will have no location data + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + + new_org_unit_location = Point(1, 2, 3) + non_deleted_instances = [self.instance_1, self.instance_5, self.instance_6, self.instance_8] + for instance in non_deleted_instances: + # setting gps data for org_units whose instance was not deleted + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_5.id]) + self.assertCountEqual(response_json["warning_no_location"], [self.instance_6.id, self.instance_8.id]) + def test_check_bulk_push_gps_selected_ids_ok(self): self.client.force_authenticate(self.yoda) new_instance = self.create_form_instance( @@ -2049,9 +2087,7 @@ def test_check_bulk_push_gps_selected_ids_error(self): for instance in [self.instance_1, self.instance_2, self.instance_3]: instance.location = Point(5, 6.45, 2.33) instance.save() - self.instance_1.refresh_from_db() - self.instance_2.refresh_from_db() - self.instance_3.refresh_from_db() + instance.refresh_from_db() response = self.client.get( f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id},{self.instance_3.id}" @@ -2076,9 +2112,13 @@ def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): def test_check_bulk_push_gps_selected_ids_error_wrong_account(self): # Preparing new setup - new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) - new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) new_instance = self.create_form_instance( form=new_form, @@ -2121,13 +2161,101 @@ def test_check_bulk_push_gps_selected_ids_warning_no_location(self): ) def test_check_bulk_push_gps_selected_ids_warning_overwrite(self): - pass + # no org unit was assigned, so we select some + self.instance_1.org_unit = self.jedi_council_endor + self.instance_2.org_unit = self.jedi_council_corruscant + new_org_unit_location = Point(1, 2, 3) + new_instance_location = Point(4, 5, 6) + for instance in [self.instance_1, self.instance_2]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + # setting gps data for these instances as well, since we want to override the org_unit location + instance.location = new_instance_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_2.id]) + + def test_check_bulk_push_gps_selected_ids_warning_both(self): + # instances 1 and 5 will overwrite their org_unit location + new_instance_location = Point(4, 5, 6) + self.instance_5.org_unit = self.jedi_council_endor + self.instance_5.location = new_instance_location + self.instance_1.location = new_instance_location + + # instances 6 and 8 will have no location data + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + + new_org_unit_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_5, self.instance_6, self.instance_8]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id}," + f"{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_5.id]) + self.assertCountEqual(response_json["warning_no_location"], [self.instance_6.id, self.instance_8.id]) def test_check_bulk_push_gps_unselected_ids_ok(self): - pass + self.instance_1.location = Point(1, 2, 3) + self.instance_1.save() + self.instance_1.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_1 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_2.id},{self.instance_3.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "success") def test_check_bulk_push_gps_unselected_ids_error(self): - pass + # no org unit was assigned, so we select some + for instance in [self.instance_1, self.instance_2]: + instance.org_unit = self.jedi_council_endor + instance.location = Point(1, 2, 3) + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_1 & instance_2 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_3.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "error") + self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) @@ -2143,9 +2271,13 @@ def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): def test_check_bulk_push_gps_unselected_ids_error_wrong_account(self): # Preparing new setup - new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) - new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) new_instance = self.create_form_instance( form=new_form, @@ -2168,7 +2300,83 @@ def test_check_bulk_push_gps_unselected_ids_error_wrong_account(self): ) def test_check_bulk_push_gps_unselected_ids_warning_no_location(self): - pass + # Changing instance_1 org unit in order to be different from instance_2 + self.instance_1.org_unit = self.jedi_council_endor + self.instance_1.save() + self.instance_1.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_1 & instance_2 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_3.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_no_location"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_warning_overwrite(self): - pass + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + + # no org unit was assigned, so we select some + new_org_unit_location = Point(1, 2, 3) + new_instance_location = Point(4, 5, 6) + for instance in [self.instance_2, self.instance_3]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + # setting gps data for these instances as well, since we want to override the org_unit location + instance.location = new_instance_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_3 & instance_2 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_1.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_3.id, self.instance_2.id]) + + def test_check_bulk_push_gps_unselected_ids_warning_both(self): + # instances 1 and 5 will overwrite their org_unit location + new_instance_location = Point(4, 5, 6) + self.instance_5.org_unit = self.jedi_council_endor + self.instance_5.location = new_instance_location + self.instance_1.location = new_instance_location + + # instances 6 and 8 will have no location data + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + + new_org_unit_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_5, self.instance_6, self.instance_8]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_2.id},{self.instance_3.id},{self.instance_4.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_5.id]) + self.assertCountEqual(response_json["warning_no_location"], [self.instance_6.id, self.instance_8.id]) From 162d612bdc8f71b55606be8d9fad702d10e602e6 Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Mon, 16 Dec 2024 12:41:36 +0200 Subject: [PATCH 04/62] WIF: bulk push GPS from submissions to OrgUnits --- .../Iaso/domains/app/translations/en.json | 3 +- .../Iaso/domains/app/translations/fr.json | 3 +- .../apps/Iaso/domains/dataSources/messages.js | 3 +- .../components/PushGpsDialogComponent.tsx | 138 ++++++++++++++++++ .../apps/Iaso/domains/instances/messages.js | 18 +++ .../Iaso/domains/instances/utils/index.tsx | 31 +++- 6 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index 578be7bab2..9f19d99e21 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -115,7 +115,6 @@ "iaso.datasources.emptyProjectsError": "Please choose at least one project", "iaso.datasources.exportDataSource": "Export data source: {dataSourceName}", "iaso.datasources.geoPkg": "Import GeoPackage", - "iaso.dataSources.goToCurrentTask": "Launch and show task", "iaso.datasources.gpkg.chooseFile": "Choose file", "iaso.datasources.gpkg.explication": "Import OrgUnits from a GeoPackage file, all the OrgUnits present in the file will be updated.{breakingLine}The file must be correctly formatted.{breakingLine}", "iaso.datasources.gpkg.importTaskExplication": "The import will be processed in the background and can take a dozen minutes to complete.", @@ -536,6 +535,7 @@ "iaso.label.fromOrgunit": "(from Org Unit)", "iaso.label.geo_json": "Geo json shape", "iaso.label.geographicalData": "Geographical data", + "iaso.label.goToCurrentTask": "Launch and show task", "iaso.label.group": "Group", "iaso.label.groups": "Groups", "iaso.label.groupSet": "Group Set", @@ -660,6 +660,7 @@ "iaso.label.projects": "Projects", "iaso.label.published": "Published", "iaso.label.publishingStatus": "Publishing status", + "iaso.label.pushGpsToOrgUnits": "Push GPS from submissions to Org units", "iaso.label.quarter": "Quarter", "iaso.label.rawHtml": "Raw html", "iaso.label.reAssignInstance": "Re-assign instance", diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index 4c2e7f6b3e..2878039d5d 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -115,7 +115,6 @@ "iaso.datasources.emptyProjectsError": "Veuillez choisir au moins un projet", "iaso.datasources.exportDataSource": "Exporter la source: {dataSourceName}", "iaso.datasources.geoPkg": "Importer un GeoPackage", - "iaso.dataSources.goToCurrentTask": "Lancer et montrer la tâche", "iaso.datasources.gpkg.chooseFile": "Choissez un fichier", "iaso.datasources.gpkg.explication": "Importer des OrgUnits depuis un fichier GeoPackage. Toutes les OrgUnits présentes dans le fichier seront mises à jour.{breakingLine}Le fichier doit être correctement formatté.{breakingLine}", "iaso.datasources.gpkg.importTaskExplication": "L'import va se dérouler en arrière-plan et peut prendre une dizaine de minutes", @@ -536,6 +535,7 @@ "iaso.label.fromOrgunit": "(de l'unité d'organisation)", "iaso.label.geo_json": "Forme geo json", "iaso.label.geographicalData": "Données géographiques", + "iaso.label.goToCurrentTask": "Lancer et montrer la tâche", "iaso.label.group": "Groupe", "iaso.label.groups": "Groupes", "iaso.label.groupSet": "Ensemble de Groupes", @@ -660,6 +660,7 @@ "iaso.label.projects": "Projets", "iaso.label.published": "Publié", "iaso.label.publishingStatus": "Statut de publication", + "iaso.label.pushGpsToOrgUnits": "Transférer les GPS des soumissions vers les unités d'org.", "iaso.label.quarter": "Trimestre", "iaso.label.rawHtml": "Simple HTML", "iaso.label.reAssignInstance": "Assigner la soumission", diff --git a/hat/assets/js/apps/Iaso/domains/dataSources/messages.js b/hat/assets/js/apps/Iaso/domains/dataSources/messages.js index 3ddbcc42c4..ef4932a075 100644 --- a/hat/assets/js/apps/Iaso/domains/dataSources/messages.js +++ b/hat/assets/js/apps/Iaso/domains/dataSources/messages.js @@ -109,9 +109,8 @@ const MESSAGES = defineMessages({ id: 'iaso.dataSources.validateStatus', defaultMessage: 'Validate status', }, - goToCurrentTask: { - id: 'iaso.dataSources.goToCurrentTask', + id: 'iaso.label.goToCurrentTask', defaultMessage: 'Launch and show task', }, launch: { diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx new file mode 100644 index 0000000000..4e7ec1997f --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -0,0 +1,138 @@ +import React, { FunctionComponent } from 'react'; +import { Box, Grid, Typography } from '@mui/material'; +import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; +import MESSAGES from '../messages'; +import { createExportRequest } from '../actions'; +import { Instance } from '../types/instance'; +import { Selection } from '../../orgUnits/types/selection'; + +type Props = { + getFilters: () => void; + renderTrigger: (openDialog: boolean) => void; + selection: Selection; +}; +const PushGpsDialogComponent: FunctionComponent = ({ + getFilters, + renderTrigger, + selection, +}) => { + const [forceExport, setForceExport] = React.useState(false); + const onConfirm = closeDialog => { + const filterParams = getFilters(); + createExportRequest({ forceExport, ...filterParams }, selection).then( + () => closeDialog(), + ); + }; + const onClosed = () => { + setForceExport(false); + }; + let title = MESSAGES.export; + if (selection) { + title = { + ...MESSAGES.pushGpsToOrgUnits, + values: { + count: selection.selectCount, + }, + }; + } + const { formatMessage } = useSafeIntl(); + return ( + // @ts-ignore + renderTrigger(openDialog)} + titleMessage={title} + onConfirm={onConfirm} + confirmMessage={MESSAGES.launch} + onClosed={onClosed} + cancelMessage={MESSAGES.cancel} + maxWidth="sm" + additionalButton + additionalMessage={MESSAGES.goToCurrentTask} + onAdditionalButtonClick={onConfirm} + allowConfimAdditionalButton + > + + + + {formatMessage(MESSAGES.pushGpsWarningMessage, { + submissionCount: 5, + orgUnitCount: 10, + })} + + + + + + + Some instances don't have locations. Nothing + will be applied for those OrgUnits. + + + + + + + See all + + + + + + + See all + + + + + + + + + Some OrgUnits already have GPS coordinates. Do + you want to proceed and overwrite them? + + + + + + + See all + + + + + + + See all + + + + + + + ); +}; +export default PushGpsDialogComponent; diff --git a/hat/assets/js/apps/Iaso/domains/instances/messages.js b/hat/assets/js/apps/Iaso/domains/instances/messages.js index f3cc9b50ae..addfe1b8a4 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/messages.js +++ b/hat/assets/js/apps/Iaso/domains/instances/messages.js @@ -202,6 +202,24 @@ const MESSAGES = defineMessages({ defaultMessage: 'Export requests', id: 'iaso.label.exportRequests', }, + pushGpsToOrgUnits: { + defaultMessage: 'Push GPS from submissions to Org units', + id: 'iaso.label.pushGpsToOrgUnits', + }, + pushGpsWarningMessage: { + defaultMessage: + 'You are about to push the locations of {submissionCount} instances to {orgUnitCount} OrgUnits', + id: 'iaso.instance.pushGpsWarningMessage', + }, + + launch: { + id: 'iaso.label.launch', + defaultMessage: 'Launch', + }, + goToCurrentTask: { + id: 'iaso.label.goToCurrentTask', + defaultMessage: 'Launch and show task', + }, compare: { defaultMessage: 'Compare', id: 'iaso.label.compare', diff --git a/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx b/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx index 3c75584fe0..61d31f2201 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx @@ -1,5 +1,6 @@ import CallMade from '@mui/icons-material/CallMade'; import CompareArrowsIcon from '@mui/icons-material/CompareArrows'; +import AddLocationIcon from '@mui/icons-material/AddLocation'; import { Tooltip } from '@mui/material'; import { Column, @@ -47,6 +48,7 @@ import * as Permission from '../../../utils/permissions'; import { useCurrentUser } from '../../../utils/usersUtils'; import { InstanceMetasField } from '../components/ColumnSelect'; import { INSTANCE_METAS_FIELDS } from '../constants'; +import PushGpsDialogComponent from '../components/PushGpsDialogComponent'; const NO_VALUE = '/'; const hasNoValue: (value: string) => boolean = value => !value || value === ''; @@ -427,6 +429,7 @@ export const useInstanceVisibleColumns = ({ getDefaultCols( formIds, labelKeys, + // @ts-ignore instanceMetasFields || INSTANCE_METAS_FIELDS, periodType, ); @@ -494,6 +497,32 @@ export const useSelectionActions = ( ); return useMemo(() => { + const pushGpsAction: SelectionAction = { + icon: newSelection => ( + filters} + renderTrigger={openDialog => { + const iconDisabled = newSelection.selectCount === 0; + const iconProps = { + onClick: !iconDisabled ? openDialog : () => null, + disabled: iconDisabled, + }; + + return ( + // @ts-ignore + + ); + }} + /> + ), + label: formatMessage(MESSAGES.pushGpsToOrgUnits), + disabled: false, + }; const exportAction: SelectionAction = { icon: newSelection => ( Date: Mon, 16 Dec 2024 18:38:22 +0200 Subject: [PATCH 05/62] WIP: bulk push gps-front --- .../Iaso/domains/app/translations/en.json | 5 + .../Iaso/domains/app/translations/fr.json | 5 + .../components/PushGpsDialogComponent.tsx | 128 ++++++++++++------ .../apps/Iaso/domains/instances/messages.js | 19 ++- 4 files changed, 114 insertions(+), 43 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index 9f19d99e21..6369892f85 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -383,16 +383,19 @@ "iaso.instance.logs.versionB": "Version B", "iaso.instance.missingFile": "Cannot find an instance with a file", "iaso.instance.missingGeolocation": "Cannot find an instance with geolocation", + "iaso.instance.noGpsForSomeInstaces": "Some instances don't have locations. Nothing will be applied for those OrgUnits.", "iaso.instance.NoLocksHistory": "There's no history", "iaso.instance.org_unit": "Org unit", "iaso.instance.org_unit_type_name": "Org unit type", "iaso.instance.patchInstanceError": "An error occurred while saving submission", "iaso.instance.patchInstanceSuccesfull": "Submission saved successfully", "iaso.instance.period": "Period", + "iaso.instance.pushGpsWarningMessage": "You are about to push the locations of {submissionCount} instances to {orgUnitCount} OrgUnits", "iaso.instance.queryBuilder": "Search in submitted fields", "iaso.instance.removeLockAction": "Unlock the submission", "iaso.instance.selectedOrgUnit": "Selected Org unit", "iaso.instance.selectVersionToCompare": "Please select the version to compare", + "iaso.instance.someOrgUnitsHasAlreadyGps": "Some OrgUnits already have GPS coordinates. Do you want to proceed and overwrite them?", "iaso.instance.source_created_at": "Created on device", "iaso.instance.table.label.instanceHeaderTooltip": "Label: {label} - Name: {key}", "iaso.instance.title": "Submissions", @@ -430,6 +433,7 @@ "iaso.label.altitude": "Altitude", "iaso.label.anyGeography": "With point or territory", "iaso.label.apply": "Apply", + "iaso.label.approve": "Approve", "iaso.label.approved": "Approved", "iaso.label.assignments": "Assignments", "iaso.label.before": "Before", @@ -676,6 +680,7 @@ "iaso.label.save": "Save", "iaso.label.search": "Search", "iaso.label.see": "See", + "iaso.label.seeAll": "See all", "iaso.label.seeAllVersions": "See all versions", "iaso.label.seeChildren": "See children", "iaso.label.seeDetails": "See details", diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index 2878039d5d..9873336170 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -383,16 +383,19 @@ "iaso.instance.logs.versionB": "Version B", "iaso.instance.missingFile": "Aucun fichier trouvé", "iaso.instance.missingGeolocation": "Aucune soumission trouvée avec une localisation", + "iaso.instance.noGpsForSomeInstaces": "Certaines soumissions n'ont pas de GPS. Rien ne sera appliqué pour ces unités d'organisation.", "iaso.instance.NoLocksHistory": "Il n'y a pas d'historique", "iaso.instance.org_unit": "Unités d'organisation", "iaso.instance.org_unit_type_name": "Type d'unité d'org.", "iaso.instance.patchInstanceError": "Une erreur est survenue lors de la sauvegarde de la soumission", "iaso.instance.patchInstanceSuccesfull": "Soumission sauvée avec succès", "iaso.instance.period": "Période", + "iaso.instance.pushGpsWarningMessage": "Vous êtes sur le point de transférer les GPS des soumissions {submissionCount} vers {orgUnitCount} unités d'Org.", "iaso.instance.queryBuilder": "Rechercher dans les champs soumis", "iaso.instance.removeLockAction": "Déverrouiller la soumission", "iaso.instance.selectedOrgUnit": "Unité d'org. sélectionnée", "iaso.instance.selectVersionToCompare": "Veuillez sélectionner la version à comparer", + "iaso.instance.someOrgUnitsHasAlreadyGps": "Certaines unités d'Org. ont déjà des coordonnées GPS. Voulez-vous continuer et les écraser ?", "iaso.instance.source_created_at": "Création sur appareil", "iaso.instance.table.label.instanceHeaderTooltip": "Label: {label} - Nom: {key}", "iaso.instance.title": "Soumissions", @@ -430,6 +433,7 @@ "iaso.label.altitude": "Altitude", "iaso.label.anyGeography": "Avec point ou territoire", "iaso.label.apply": "Appliquer", + "iaso.label.approve": "Approuvez", "iaso.label.approved": "Approuvé", "iaso.label.assignments": "Assignations", "iaso.label.before": "Avant", @@ -676,6 +680,7 @@ "iaso.label.save": "Sauver", "iaso.label.search": "Recherche", "iaso.label.see": "Voir", + "iaso.label.seeAll": "Voir tout", "iaso.label.seeAllVersions": "Voir toutes les versions", "iaso.label.seeChildren": "Voir les enfants", "iaso.label.seeDetails": "Voir les détails", diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 4e7ec1997f..9029357b64 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -1,5 +1,5 @@ -import React, { FunctionComponent } from 'react'; -import { Box, Grid, Typography } from '@mui/material'; +import React, { FunctionComponent, useState } from 'react'; +import { Button, Grid, Typography } from '@mui/material'; import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; @@ -17,7 +17,11 @@ const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger, selection, }) => { - const [forceExport, setForceExport] = React.useState(false); + const [forceExport, setForceExport] = useState(false); + const [approveOrgUnitHasGps, setApproveOrgUnitHasGps] = + useState(false); + const [approveSubmissionNoHasGps, setApproveSubmissionNoHasGps] = + useState(false); const onConfirm = closeDialog => { const filterParams = getFilters(); createExportRequest({ forceExport, ...filterParams }, selection).then( @@ -27,6 +31,24 @@ const PushGpsDialogComponent: FunctionComponent = ({ const onClosed = () => { setForceExport(false); }; + + const onApprove = type => { + if (type === 'instanceNoGps') { + if (approveSubmissionNoHasGps) { + setApproveSubmissionNoHasGps(false); + } else { + setApproveSubmissionNoHasGps(true); + } + } + + if (type === 'orgUnitHasGps') { + if (approveOrgUnitHasGps) { + setApproveOrgUnitHasGps(false); + } else { + setApproveOrgUnitHasGps(true); + } + } + }; let title = MESSAGES.export; if (selection) { title = { @@ -52,13 +74,9 @@ const PushGpsDialogComponent: FunctionComponent = ({ onAdditionalButtonClick={onConfirm} allowConfimAdditionalButton > - + - + {formatMessage(MESSAGES.pushGpsWarningMessage, { submissionCount: 5, orgUnitCount: 10, @@ -73,31 +91,43 @@ const PushGpsDialogComponent: FunctionComponent = ({ alignItems="center" direction="row" > - + - Some instances don't have locations. Nothing - will be applied for those OrgUnits. + {formatMessage(MESSAGES.noGpsForSomeInstaces)} - - - - See all - - + + + {formatMessage(MESSAGES.seeAll)} + - - - - See all - - + + = ({ alignItems="center" direction="row" > - - + + - Some OrgUnits already have GPS coordinates. Do - you want to proceed and overwrite them? + {formatMessage( + MESSAGES.someOrgUnitsHasAlreadyGps, + )} - - - - See all - - + + + {formatMessage(MESSAGES.seeAll)} + - - - - See all - - + + diff --git a/hat/assets/js/apps/Iaso/domains/instances/messages.js b/hat/assets/js/apps/Iaso/domains/instances/messages.js index addfe1b8a4..ef954b5766 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/messages.js +++ b/hat/assets/js/apps/Iaso/domains/instances/messages.js @@ -211,7 +211,24 @@ const MESSAGES = defineMessages({ 'You are about to push the locations of {submissionCount} instances to {orgUnitCount} OrgUnits', id: 'iaso.instance.pushGpsWarningMessage', }, - + noGpsForSomeInstaces: { + defaultMessage: + "Some instances don't have locations. Nothing will be applied for those OrgUnits.", + id: 'iaso.instance.noGpsForSomeInstaces', + }, + someOrgUnitsHasAlreadyGps: { + defaultMessage: + 'Some OrgUnits already have GPS coordinates. Do you want to proceed and overwrite them?', + id: 'iaso.instance.someOrgUnitsHasAlreadyGps', + }, + seeAll: { + defaultMessage: 'See all', + id: 'iaso.label.seeAll', + }, + approve: { + defaultMessage: 'Approve', + id: 'iaso.label.approve', + }, launch: { id: 'iaso.label.launch', defaultMessage: 'Launch', From ede85f06c0c4d46262aa7519845250c3c1f0792b Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Tue, 17 Dec 2024 15:23:10 +0200 Subject: [PATCH 06/62] get warnings and status from backend --- .../components/PushGpsDialogComponent.tsx | 200 ++++++++++-------- .../hooks/useGetCheckBulkGpsPush.tsx | 28 +++ .../Iaso/domains/instances/types/instance.ts | 5 + .../Iaso/domains/instances/utils/index.tsx | 1 - 4 files changed, 145 insertions(+), 89 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 9029357b64..58fb55b691 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -3,33 +3,39 @@ import { Button, Grid, Typography } from '@mui/material'; import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; -import { createExportRequest } from '../actions'; import { Instance } from '../types/instance'; import { Selection } from '../../orgUnits/types/selection'; +import { useGetCheckBulkGpsPush } from '../hooks/useGetCheckBulkGpsPush'; type Props = { - getFilters: () => void; renderTrigger: (openDialog: boolean) => void; selection: Selection; }; const PushGpsDialogComponent: FunctionComponent = ({ - getFilters, renderTrigger, selection, }) => { - const [forceExport, setForceExport] = useState(false); const [approveOrgUnitHasGps, setApproveOrgUnitHasGps] = useState(false); const [approveSubmissionNoHasGps, setApproveSubmissionNoHasGps] = useState(false); const onConfirm = closeDialog => { - const filterParams = getFilters(); - createExportRequest({ forceExport, ...filterParams }, selection).then( - () => closeDialog(), - ); + return null; }; + + const { data: checkBulkGpsPush, isFetching: isLoadingCheckBulkGpsPush } = + useGetCheckBulkGpsPush({ + selected_ids: selection.selectedItems + .map(item => item.id) + .join(','), + select_all: selection.selectAll, + unselected_ids: selection.unSelectedItems + .map(item => item.id) + .join(','), + }); + const onClosed = () => { - setForceExport(false); + return null; }; const onApprove = type => { @@ -83,98 +89,116 @@ const PushGpsDialogComponent: FunctionComponent = ({ })} - - - - - {formatMessage(MESSAGES.noGpsForSomeInstaces)} - - - + {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && ( - - {formatMessage(MESSAGES.seeAll)} - - - - - - - - - + {formatMessage(MESSAGES.seeAll)} + + + - + + + )} + {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && ( - - {formatMessage(MESSAGES.seeAll)} - - - - + + {formatMessage(MESSAGES.seeAll)} + + + + + - + )} ); diff --git a/hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx b/hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx new file mode 100644 index 0000000000..3783fab996 --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx @@ -0,0 +1,28 @@ +import { UseQueryResult } from 'react-query'; +import { useSnackQuery } from '../../../libs/apiHooks'; +import MESSAGES from '../messages'; +import { getRequest } from '../../../libs/Api'; +import { makeUrlWithParams } from '../../../libs/utils'; +import { CheckBulkGpsPushResult } from '../types/instance'; + +export const useGetCheckBulkGpsPush = ( + params, +): UseQueryResult => { + return useSnackQuery({ + queryKey: ['bulkGpsCheck', params], + queryFn: () => { + return getRequest( + makeUrlWithParams( + '/api/instances/check_bulk_gps_push/', + params, + ), + ); + }, + snackErrorMsg: MESSAGES.error, + options: { + select: data => { + return data; + }, + }, + }); +}; diff --git a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts index dc06113091..af1bcaca0f 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts @@ -104,6 +104,11 @@ export interface PaginatedInstances extends Pagination { instances: Instance[]; } +export type CheckBulkGpsPushResult = { + result: string; + warning_no_location?: number[]; + warning_overwrite?: number[]; +}; export type MimeType = // Text | 'text/plain' diff --git a/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx b/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx index 61d31f2201..653406fa51 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx @@ -502,7 +502,6 @@ export const useSelectionActions = ( filters} renderTrigger={openDialog => { const iconDisabled = newSelection.selectCount === 0; const iconProps = { From ce8014a52982646c8c8b1ca0d950570e34a9e1f9 Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Tue, 17 Dec 2024 18:09:25 +0200 Subject: [PATCH 07/62] WIP --- .../Iaso/domains/app/translations/en.json | 3 +- .../Iaso/domains/app/translations/fr.json | 3 +- .../components/PushGpsDialogComponent.tsx | 210 +++++++++--------- .../apps/Iaso/domains/instances/messages.js | 4 + .../Iaso/domains/instances/types/instance.ts | 1 + 5 files changed, 118 insertions(+), 103 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index 21c9e460e2..c82f2d319b 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -383,6 +383,7 @@ "iaso.instance.logs.versionB": "Version B", "iaso.instance.missingFile": "Cannot find an instance with a file", "iaso.instance.missingGeolocation": "Cannot find an instance with geolocation", + "iaso.instance.multipleInstancesOneOrgUnitWarningMessage": "Multiple submissions are using the same org unit", "iaso.instance.noGpsForSomeInstaces": "Some instances don't have locations. Nothing will be applied for those OrgUnits.", "iaso.instance.NoLocksHistory": "There's no history", "iaso.instance.org_unit": "Org unit", @@ -1583,4 +1584,4 @@ "trypelim.permissions.zones": "Zones", "trypelim.permissions.zones_edit": "Edit zones", "trypelim.permissions.zones_shapes_edit": "Edit zone shapes" -} +} \ No newline at end of file diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index 30d9ac2d4d..62d9d1981f 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -383,6 +383,7 @@ "iaso.instance.logs.versionB": "Version B", "iaso.instance.missingFile": "Aucun fichier trouvé", "iaso.instance.missingGeolocation": "Aucune soumission trouvée avec une localisation", + "iaso.instance.multipleInstancesOneOrgUnitWarningMessage": "Plusieurs soumissions utilisent la même unité d'org.", "iaso.instance.noGpsForSomeInstaces": "Certaines soumissions n'ont pas de GPS. Rien ne sera appliqué pour ces unités d'organisation.", "iaso.instance.NoLocksHistory": "Il n'y a pas d'historique", "iaso.instance.org_unit": "Unités d'organisation", @@ -1582,4 +1583,4 @@ "trypelim.permissions.zones": "Zones", "trypelim.permissions.zones_edit": "Edit zones", "trypelim.permissions.zones_shapes_edit": "Editer les contours géographiques des zones de santé" -} +} \ No newline at end of file diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 58fb55b691..43a93ba9ab 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -83,122 +83,130 @@ const PushGpsDialogComponent: FunctionComponent = ({ - {formatMessage(MESSAGES.pushGpsWarningMessage, { - submissionCount: 5, - orgUnitCount: 10, - })} + {(checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 + ? formatMessage(MESSAGES.pushGpsWarningMessage, { + submissionCount: selection.selectCount, + orgUnitCount: selection.selectCount, + }) + : formatMessage( + MESSAGES.multipleInstancesOneOrgUnitWarningMessage, + )} - {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && ( - - - - - {formatMessage( - MESSAGES.noGpsForSomeInstaces, - )} - - - + {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && + (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - - {formatMessage(MESSAGES.seeAll)} - - - - - - - )} - {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && ( - - - + {formatMessage(MESSAGES.seeAll)} + + + - + + + )} + {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && + (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - + + - - )} + )} ); diff --git a/hat/assets/js/apps/Iaso/domains/instances/messages.js b/hat/assets/js/apps/Iaso/domains/instances/messages.js index ef954b5766..a1d8be39cc 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/messages.js +++ b/hat/assets/js/apps/Iaso/domains/instances/messages.js @@ -221,6 +221,10 @@ const MESSAGES = defineMessages({ 'Some OrgUnits already have GPS coordinates. Do you want to proceed and overwrite them?', id: 'iaso.instance.someOrgUnitsHasAlreadyGps', }, + multipleInstancesOneOrgUnitWarningMessage: { + defaultMessage: 'Multiple submissions are using the same org unit', + id: 'iaso.instance.multipleInstancesOneOrgUnitWarningMessage', + }, seeAll: { defaultMessage: 'See all', id: 'iaso.label.seeAll', diff --git a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts index af1bcaca0f..996a34a3e1 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts @@ -108,6 +108,7 @@ export type CheckBulkGpsPushResult = { result: string; warning_no_location?: number[]; warning_overwrite?: number[]; + error_ids?: number[]; }; export type MimeType = // Text From acd345e5ffadfe5177973645fe5ee0b507ea2552 Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Thu, 19 Dec 2024 10:51:42 +0200 Subject: [PATCH 08/62] refactor the code --- .../components/PushGpsDialogComponent.tsx | 214 +++++++----------- 1 file changed, 81 insertions(+), 133 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 43a93ba9ab..cde93c7737 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -11,6 +11,54 @@ type Props = { renderTrigger: (openDialog: boolean) => void; selection: Selection; }; + +const WarningSection = ({ + condition, + message, + linkTo, + approveCondition, + onApproveClick, +}) => { + const { formatMessage } = useSafeIntl(); + if (!condition) return null; + + return ( + + + + + {formatMessage(message)} + + + + + + {formatMessage(MESSAGES.seeAll)} + + + + + + + ); +}; + const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger, selection, @@ -23,16 +71,13 @@ const PushGpsDialogComponent: FunctionComponent = ({ return null; }; - const { data: checkBulkGpsPush, isFetching: isLoadingCheckBulkGpsPush } = - useGetCheckBulkGpsPush({ - selected_ids: selection.selectedItems - .map(item => item.id) - .join(','), - select_all: selection.selectAll, - unselected_ids: selection.unSelectedItems - .map(item => item.id) - .join(','), - }); + const { data: checkBulkGpsPush, isError } = useGetCheckBulkGpsPush({ + selected_ids: selection.selectedItems.map(item => item.id).join(','), + select_all: selection.selectAll, + unselected_ids: selection.unSelectedItems + .map(item => item.id) + .join(','), + }); const onClosed = () => { return null; @@ -59,9 +104,6 @@ const PushGpsDialogComponent: FunctionComponent = ({ if (selection) { title = { ...MESSAGES.pushGpsToOrgUnits, - values: { - count: selection.selectCount, - }, }; } const { formatMessage } = useSafeIntl(); @@ -83,130 +125,36 @@ const PushGpsDialogComponent: FunctionComponent = ({ - {(checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 - ? formatMessage(MESSAGES.pushGpsWarningMessage, { + {isError + ? formatMessage( + MESSAGES.multipleInstancesOneOrgUnitWarningMessage, + ) + : formatMessage(MESSAGES.pushGpsWarningMessage, { submissionCount: selection.selectCount, orgUnitCount: selection.selectCount, - }) - : formatMessage( - MESSAGES.multipleInstancesOneOrgUnitWarningMessage, - )} + })} - {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && - (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - - - - - {formatMessage( - MESSAGES.noGpsForSomeInstaces, - )} - - - - - - {formatMessage(MESSAGES.seeAll)} - - - - - - - )} - {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && - (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - - - - - {formatMessage( - MESSAGES.someOrgUnitsHasAlreadyGps, - )} - - - - - - {formatMessage(MESSAGES.seeAll)} - - - - - - - )} + + 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 + } + message={MESSAGES.noGpsForSomeInstaces} + linkTo="url" + approveCondition={approveSubmissionNoHasGps} + onApproveClick={() => onApprove('instanceNoGps')} + /> + + 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 + } + message={MESSAGES.someOrgUnitsHasAlreadyGps} + linkTo="url" + approveCondition={approveOrgUnitHasGps} + onApproveClick={() => onApprove('orgUnitHasGps')} + /> ); From c99811973d1c8aac05a736ad9d4e7da0fb5a5515 Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Fri, 20 Dec 2024 10:20:37 +0200 Subject: [PATCH 09/62] Move warning code into different file --- .../components/PushBulkGpsWarning.tsx | 60 +++++++++++++++++++ .../components/PushGpsDialogComponent.tsx | 56 ++--------------- 2 files changed, 65 insertions(+), 51 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx new file mode 100644 index 0000000000..53114365ea --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx @@ -0,0 +1,60 @@ +import { Button, Grid, Typography } from '@mui/material'; +import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import React, { FunctionComponent } from 'react'; +import MESSAGES from '../messages'; + +type Props = { + condition: boolean; + message: { defaultMessage: string; id: string }; + linkTo: string; + approveCondition: boolean; + onApproveClick: () => void; +}; +const PushBulkGpsWarning: FunctionComponent = ({ + condition, + message, + linkTo, + approveCondition, + onApproveClick, +}) => { + const { formatMessage } = useSafeIntl(); + if (!condition) return null; + + return ( + + + + + {formatMessage(message)} + + + + + + {formatMessage(MESSAGES.seeAll)} + + + + + + + ); +}; + +export default PushBulkGpsWarning; diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index cde93c7737..38ec59fb04 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -1,64 +1,18 @@ import React, { FunctionComponent, useState } from 'react'; -import { Button, Grid, Typography } from '@mui/material'; -import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import { Grid, Typography } from '@mui/material'; +import { useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; import { Instance } from '../types/instance'; import { Selection } from '../../orgUnits/types/selection'; import { useGetCheckBulkGpsPush } from '../hooks/useGetCheckBulkGpsPush'; +import PushBulkGpsWarning from './PushBulkGpsWarning'; type Props = { renderTrigger: (openDialog: boolean) => void; selection: Selection; }; -const WarningSection = ({ - condition, - message, - linkTo, - approveCondition, - onApproveClick, -}) => { - const { formatMessage } = useSafeIntl(); - if (!condition) return null; - - return ( - - - - - {formatMessage(message)} - - - - - - {formatMessage(MESSAGES.seeAll)} - - - - - - - ); -}; - const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger, selection, @@ -135,7 +89,7 @@ const PushGpsDialogComponent: FunctionComponent = ({ })} - 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 @@ -145,7 +99,7 @@ const PushGpsDialogComponent: FunctionComponent = ({ approveCondition={approveSubmissionNoHasGps} onApproveClick={() => onApprove('instanceNoGps')} /> - 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 From 0dc467753d9c6fba26bde8cd3781904c06330ef3 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 20 Dec 2024 12:28:46 +0100 Subject: [PATCH 10/62] Added task instance_bulk_gps_push Refactored the bulk gps check in order to call it inside the task. Fixed typo in HasCreateOrgUnitPermission. --- hat/audit/models.py | 1 + iaso/api/instances.py | 81 ++-- iaso/api/org_units.py | 6 +- .../tasks/create/instance_bulk_gps_push.py | 33 ++ iaso/tasks/instance_bulk_gps_push.py | 76 ++++ iaso/tests/api/test_instances.py | 80 +++- .../tasks/test_instance_bulk_gps_push.py | 400 ++++++++++++++++++ iaso/urls.py | 2 + iaso/utils/models/common.py | 77 ++++ 9 files changed, 692 insertions(+), 64 deletions(-) create mode 100644 iaso/api/tasks/create/instance_bulk_gps_push.py create mode 100644 iaso/tasks/instance_bulk_gps_push.py create mode 100644 iaso/tests/tasks/test_instance_bulk_gps_push.py diff --git a/hat/audit/models.py b/hat/audit/models.py index f97f0e367b..c323d6e1a0 100644 --- a/hat/audit/models.py +++ b/hat/audit/models.py @@ -16,6 +16,7 @@ ORG_UNIT_API_BULK = "org_unit_api_bulk" GROUP_SET_API = "group_set_api" INSTANCE_API = "instance_api" +INSTANCE_API_BULK = "instance_api_bulk" FORM_API = "form_api" GPKG_IMPORT = "gpkg_import" CAMPAIGN_API = "campaign_api" diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 8f87b5a8ea..86eefd8d62 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -38,9 +38,10 @@ ) from iaso.utils import timestamp_to_datetime from iaso.utils.file_utils import get_file_type +from .org_units import HasCreateOrgUnitPermission from ..models.forms import CR_MODE_IF_REFERENCE_FORM -from ..utils.models.common import get_creator_name +from ..utils.models.common import get_creator_name, check_instance_bulk_gps_push from . import common from .comment import UserSerializerForComment from .common import ( @@ -91,7 +92,7 @@ def validate_period(self, value): class HasInstancePermission(permissions.BasePermission): def has_permission(self, request: Request, view): - if request.method == "POST": + if request.method == "POST": # to handle anonymous submissions sent by mobile return True return request.user.is_authenticated and ( @@ -112,6 +113,20 @@ def has_object_permission(self, request: Request, view, obj: Instance): return False +class HasInstanceBulkPermission(permissions.BasePermission): + """ + Designed for POST endpoints that are not designed to receive new submissions. + """ + + def has_permission(self, request: Request, view): + return request.user.is_authenticated and ( + request.user.has_perm(permission.FORMS) + or request.user.has_perm(permission.SUBMISSIONS) + or request.user.has_perm(permission.REGISTRY_WRITE) + or request.user.has_perm(permission.REGISTRY_READ) + ) + + class InstanceFileSerializer(serializers.Serializer): id = serializers.IntegerField(read_only=True) instance_id = serializers.IntegerField() @@ -605,7 +620,11 @@ def bulkdelete(self, request): status=201, ) - @action(detail=False, methods=["GET"], permission_classes=[permissions.IsAuthenticated, HasInstancePermission]) + @action( + detail=False, + methods=["GET"], + permission_classes=[permissions.IsAuthenticated, HasInstanceBulkPermission, HasCreateOrgUnitPermission], + ) def check_bulk_gps_push(self, request): # first, let's parse all parameters received from the URL select_all, selected_ids, unselected_ids = self._parse_check_bulk_gps_push_parameters(request.GET) @@ -628,48 +647,15 @@ def check_bulk_gps_push(self, request): else: instances_query = instances_query.exclude(pk__in=unselected_ids) - overwrite_ids = [] - no_location_ids = [] - org_units_to_instances_dict = {} - set_org_units_ids = set() + success, errors, warnings = check_instance_bulk_gps_push(instances_query) - for instance in instances_query: - if not instance.location: - no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit - continue - - org_unit = instance.org_unit - if org_unit.id in org_units_to_instances_dict: - # we can't push this instance's location since there was another instance linked to this OrgUnit - org_units_to_instances_dict[org_unit.id].append(instance.id) - continue - else: - org_units_to_instances_dict[org_unit.id] = [instance.id] - - set_org_units_ids.add(org_unit.id) - if org_unit.location or org_unit.geom: - overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location - continue - - # Before returning, we need to check if we've had multiple hits on an OrgUnit - error_same_org_unit_ids = self._check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + if not success: + errors["result"] = "errors" + return Response(errors, status=status.HTTP_400_BAD_REQUEST) - if len(error_same_org_unit_ids): - return Response( - {"result": "error", "error_ids": error_same_org_unit_ids}, - status=status.HTTP_400_BAD_REQUEST, - ) - - if len(no_location_ids) or len(overwrite_ids): - dict_response = { - "result": "warnings", - } - if len(no_location_ids): - dict_response["warning_no_location"] = no_location_ids - if len(overwrite_ids): - dict_response["warning_overwrite"] = overwrite_ids - - return Response(dict_response, status=status.HTTP_200_OK) + if warnings: + warnings["result"] = "warnings" + return Response(warnings, status=status.HTTP_200_OK) return Response( { @@ -680,7 +666,7 @@ def check_bulk_gps_push(self, request): def _parse_check_bulk_gps_push_parameters(self, query_parameters): raw_select_all = query_parameters.get("select_all", True) - select_all = raw_select_all not in ["false", "False", "0"] + select_all = raw_select_all not in ["false", "False", "0", 0, False] raw_selected_ids = query_parameters.get("selected_ids", None) if raw_selected_ids: @@ -696,13 +682,6 @@ def _parse_check_bulk_gps_push_parameters(self, query_parameters): return select_all, selected_ids, unselected_ids - def _check_bulk_gps_repeated_org_units(self, org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: - error_instance_ids = [] - for _, instance_ids in org_units_to_instance_ids.items(): - if len(instance_ids) >= 2: - error_instance_ids.extend(instance_ids) - return error_instance_ids - QUERY = """ select DATE_TRUNC('month', COALESCE(iaso_instance.source_created_at, iaso_instance.created_at)) as month, (select name from iaso_form where id = iaso_instance.form_id) as form_name, diff --git a/iaso/api/org_units.py b/iaso/api/org_units.py index 45623ff7de..e28568a20f 100644 --- a/iaso/api/org_units.py +++ b/iaso/api/org_units.py @@ -32,7 +32,7 @@ # noinspection PyMethodMayBeStatic -class HasCreateOrUnitPermission(permissions.BasePermission): +class HasCreateOrgUnitPermission(permissions.BasePermission): def has_permission(self, request, view): if not request.user.is_authenticated: return False @@ -614,7 +614,9 @@ def get_date(self, date: str) -> Union[datetime.date, None]: pass return None - @action(detail=False, methods=["POST"], permission_classes=[permissions.IsAuthenticated, HasCreateOrUnitPermission]) + @action( + detail=False, methods=["POST"], permission_classes=[permissions.IsAuthenticated, HasCreateOrgUnitPermission] + ) def create_org_unit(self, request): """This endpoint is used by the React frontend""" errors = [] diff --git a/iaso/api/tasks/create/instance_bulk_gps_push.py b/iaso/api/tasks/create/instance_bulk_gps_push.py new file mode 100644 index 0000000000..3ba956269c --- /dev/null +++ b/iaso/api/tasks/create/instance_bulk_gps_push.py @@ -0,0 +1,33 @@ +from rest_framework import viewsets, permissions, status +from rest_framework.response import Response + +from iaso.api.instances import HasInstanceBulkPermission +from iaso.api.org_units import HasCreateOrgUnitPermission +from iaso.api.tasks import TaskSerializer +from iaso.tasks.instance_bulk_gps_push import instance_bulk_gps_push + + +class InstanceBulkGpsPushViewSet(viewsets.ViewSet): + """Bulk push gps location from Instances to their related OrgUnit. + + This task will override existing location on OrgUnits and might set `None` if the Instance doesn't have any location. + Calling this endpoint implies that the InstanceViewSet.check_bulk_gps_push() method has been called before and has returned no error. + """ + + permission_classes = [permissions.IsAuthenticated, HasInstanceBulkPermission, HasCreateOrgUnitPermission] + + def create(self, request): + raw_select_all = request.data.get("select_all", True) + select_all = raw_select_all not in [False, "false", "False", "0", 0] + selected_ids = request.data.get("selected_ids", []) + unselected_ids = request.data.get("unselected_ids", []) + + user = self.request.user + + task = instance_bulk_gps_push( + select_all=select_all, selected_ids=selected_ids, unselected_ids=unselected_ids, user=user + ) + return Response( + {"task": TaskSerializer(instance=task).data}, + status=status.HTTP_201_CREATED, + ) diff --git a/iaso/tasks/instance_bulk_gps_push.py b/iaso/tasks/instance_bulk_gps_push.py new file mode 100644 index 0000000000..53ac920d33 --- /dev/null +++ b/iaso/tasks/instance_bulk_gps_push.py @@ -0,0 +1,76 @@ +from copy import deepcopy +from logging import getLogger +from time import time +from typing import Optional, List + +from django.contrib.auth.models import User +from django.db import transaction + +from beanstalk_worker import task_decorator +from hat.audit import models as audit_models +from iaso.models import Task, Instance +from iaso.utils.gis import convert_2d_point_to_3d +from iaso.utils.models.common import check_instance_bulk_gps_push + +logger = getLogger(__name__) + + +def push_single_instance_gps_to_org_unit(user: Optional[User], instance: Instance): + org_unit = instance.org_unit + original_copy = deepcopy(org_unit) + org_unit.location = convert_2d_point_to_3d(instance.location) if instance.location else None + org_unit.save() + if not original_copy.location: + logger.info(f"updating {org_unit.name} {org_unit.id} with {org_unit.location}") + else: + logger.info( + f"updating {org_unit.name} {org_unit.id} - overwriting {original_copy.location} with {org_unit.location}" + ) + audit_models.log_modification(original_copy, org_unit, source=audit_models.INSTANCE_API_BULK, user=user) + + +@task_decorator(task_name="instance_bulk_gps_push") +def instance_bulk_gps_push( + select_all: bool, + selected_ids: List[int], + unselected_ids: List[int], + task: Task, +): + """Background Task to bulk push instance gps to org units. + + /!\ Danger: calling this task without having received a successful response from the check_bulk_gps_push + endpoint will have unexpected results that might cause data loss. + """ + start = time() + task.report_progress_and_stop_if_killed(progress_message="Searching for Instances for pushing gps data") + + user = task.launcher + + queryset = Instance.non_deleted_objects.get_queryset().filter_for_user(user) + queryset = queryset.select_related("org_unit") + + if not select_all: + queryset = queryset.filter(pk__in=selected_ids) + else: + queryset = queryset.exclude(pk__in=unselected_ids) + + if not queryset: + raise Exception("No matching instances found") + + # Checking if any gps push can be performed with what was requested + success, errors, _ = check_instance_bulk_gps_push(queryset) + if not success: + raise Exception("Cannot proceed with the gps push due to errors: %s" % errors) + + total = queryset.count() + + with transaction.atomic(): + for index, instance in enumerate(queryset.iterator()): + res_string = "%.2f sec, processed %i instances" % (time() - start, index) + task.report_progress_and_stop_if_killed(progress_message=res_string, end_value=total, progress_value=index) + push_single_instance_gps_to_org_unit( + user, + instance, + ) + + task.report_success(message="%d modified" % total) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index e7defdfddc..11cf8256f3 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -39,7 +39,11 @@ def setUpTestData(cls): cls.sw_version = sw_version cls.yoda = cls.create_user_with_profile( - username="yoda", last_name="Da", first_name="Yo", account=star_wars, permissions=["iaso_submissions"] + username="yoda", + last_name="Da", + first_name="Yo", + account=star_wars, + permissions=["iaso_submissions", "iaso_org_units"], ) cls.guest = cls.create_user_with_profile(username="guest", account=star_wars, permissions=["iaso_submissions"]) cls.supervisor = cls.create_user_with_profile( @@ -72,10 +76,15 @@ def setUpTestData(cls): version=sw_version, ) cls.jedi_council_endor = m.OrgUnit.objects.create( - name="Endor Jedi Council", source_ref="jedi_council_endor_ref" + name="Endor Jedi Council", + source_ref="jedi_council_endor_ref", + version=sw_version, ) cls.jedi_council_endor_region = m.OrgUnit.objects.create( - name="Endor Region Jedi Council", parent=cls.jedi_council_endor, source_ref="jedi_council_endor_region_ref" + name="Endor Region Jedi Council", + parent=cls.jedi_council_endor, + source_ref="jedi_council_endor_region_ref", + version=sw_version, ) cls.project = m.Project.objects.create( @@ -1963,8 +1972,8 @@ def test_check_bulk_push_gps_select_all_ok(self): response_json = response.json() self.assertEqual(response_json["result"], "success") - def test_check_bulk_push_gps_select_all_error(self): - # setting gps data for instances that were not deleted + def test_check_bulk_push_gps_select_all_error_same_org_unit(self): + # changing location for some instances to have multiple hits on multiple org_units self.instance_1.org_unit = self.jedi_council_endor self.instance_2.org_unit = self.jedi_council_endor new_location = Point(1, 2, 3) @@ -1973,14 +1982,61 @@ def test_check_bulk_push_gps_select_all_error(self): instance.save() instance.refresh_from_db() + # Let's delete some instances, the result will be the same + for instance in [self.instance_6, self.instance_8]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + self.client.force_authenticate(self.yoda) response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual( + response_json["error_same_org_unit"], + [self.instance_1.id, self.instance_2.id, self.instance_3.id, self.instance_4.id, self.instance_5.id], + ) + + def test_check_bulk_push_gps_select_all_error_read_only_source(self): + # Making the source read only + self.sw_source.read_only = True + self.sw_source.save() + + # Changing some instance.org_unit so that all the results don't appear only in "error_same_org_unit" + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + for instance in [self.instance_2, self.instance_3, self.instance_8]: + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "errors") + # instance_6 included because it's the first one with the remaining org_unit and the queryset has a default order of "-id" + self.assertCountEqual( + response_json["error_read_only_source"], + [self.instance_8.id, self.instance_2.id, self.instance_3.id, self.instance_6.id], + ) def test_check_bulk_push_gps_select_all_warning_no_location(self): + # Changing some instance.org_unit so that all the results don't appear only in "error_same_org_unit" + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + for instance in [self.instance_2, self.instance_3, self.instance_8]: + instance.save() + instance.refresh_from_db() + + # Let's delete some instances to avoid getting "error_same_org-unit" + for instance in [self.instance_4, self.instance_5, self.instance_6, self.instance_8]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + self.client.force_authenticate(self.yoda) response = self.client.get(f"/api/instances/check_bulk_gps_push/") self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -2095,8 +2151,10 @@ def test_check_bulk_push_gps_selected_ids_error(self): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() # All these Instances target the same OrgUnit, so it's impossible to push gps data - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual( + response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id, self.instance_3.id] + ) def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) @@ -2254,8 +2312,8 @@ def test_check_bulk_push_gps_unselected_ids_error(self): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual(response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) diff --git a/iaso/tests/tasks/test_instance_bulk_gps_push.py b/iaso/tests/tasks/test_instance_bulk_gps_push.py new file mode 100644 index 0000000000..0910baeb1f --- /dev/null +++ b/iaso/tests/tasks/test_instance_bulk_gps_push.py @@ -0,0 +1,400 @@ +from django.contrib.auth.models import Permission +from django.contrib.contenttypes.models import ContentType +from django.contrib.gis.geos import Point +from rest_framework import status + +from hat.menupermissions import models as am +from iaso import models as m +from iaso.models import Task, QUEUED +from iaso.tests.tasks.task_api_test_case import TaskAPITestCase + + +class InstanceBulkPushGpsAPITestCase(TaskAPITestCase): + BASE_URL = "/api/tasks/create/instancebulkgpspush/" + + @classmethod + def setUpTestData(cls): + # Preparing account, data source, project, users... + cls.account, cls.data_source, cls.source_version, cls.project = cls.create_account_datasource_version_project( + "source", "account", "project" + ) + cls.user, cls.anon_user, cls.user_no_perms = cls.create_base_users( + cls.account, ["iaso_submissions", "iaso_org_units"] + ) + + # Preparing org units & locations + cls.org_unit_type = m.OrgUnitType.objects.create(name="Org unit type", short_name="OUT") + cls.org_unit_type.projects.add(cls.project) + cls.org_unit_no_location = m.OrgUnit.objects.create( + name="No location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + cls.default_location = Point(x=4, y=50, z=100, srid=4326) + cls.other_location = Point(x=2, y=-50, z=100, srid=4326) + cls.org_unit_with_default_location = m.OrgUnit.objects.create( + name="Default location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + location=cls.default_location, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + cls.org_unit_with_other_location = m.OrgUnit.objects.create( + name="Other location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + location=cls.other_location, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + + # Preparing instances - all linked to org_unit_without_location + cls.form = m.Form.objects.create(name="form", period_type=m.MONTH, single_per_period=True) + cls.instance_without_location = cls.create_form_instance( + form=cls.form, + period="202001", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="noLoc", + ) + cls.instance_with_default_location = cls.create_form_instance( + form=cls.form, + period="202002", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="defaultLoc", + location=cls.default_location, + ) + cls.instance_with_other_location = cls.create_form_instance( + form=cls.form, + period="202003", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="otherLoc", + location=cls.other_location, + ) + + def test_ok(self): + """POST /api/tasks/create/instancebulkgpspush/ without any error nor warning""" + + # Setting up one more instance and orgunit + new_org_unit = m.OrgUnit.objects.create( + name="new org unit", + org_unit_type=self.org_unit_type, + validation_status=m.OrgUnit.VALIDATION_VALID, + version=self.source_version, + source_ref="new org unit", + ) + new_instance = m.Instance.objects.create( + org_unit=new_org_unit, + form=self.form, + period="202004", + project=self.project, + created_by=self.user, + export_id="instance4", + ) + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_without_location.id, new_instance.id], + }, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_201_CREATED) + + def test_not_logged_in(self): + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_401_UNAUTHORIZED) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_no_permission_instances(self): + """POST /api/tasks/create/instancebulkgpspush/ without instances permissions""" + # Adding org unit permission to user + content_type = ContentType.objects.get_for_model(am.CustomPermissionSupport) + self.user_no_perms.user_permissions.add( + Permission.objects.filter(codename="iaso_org_units", content_type=content_type).first().id + ) + + self.client.force_authenticate(self.user_no_perms) + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_no_permission_org_units(self): + """POST /api/tasks/create/instancebulkgpspush/ without orgunit permissions""" + # Adding instances permission to user + content_type = ContentType.objects.get_for_model(am.CustomPermissionSupport) + self.user_no_perms.user_permissions.add( + Permission.objects.filter(codename="iaso_submissions", content_type=content_type).first().id + ) + + self.client.force_authenticate(self.user_no_perms) + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_instance_ids_wrong_account(self): + """POST /api/tasks/create/instancebulkgpspush/ with instance IDs from another account""" + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) + new_user = self.create_user_with_profile( + username="new user", account=new_account, permissions=["iaso_submissions", "iaso_org_units"] + ) + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + _ = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + location=Point(1, 2, 3, 4326), + ) + + self.client.force_authenticate(new_user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_without_location.id, self.instance_with_default_location.id], + }, + format="json", + ) + + # Task is successfully created but will fail once it starts + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, new_user) + + # Let's run the task to see the error + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + self.assertEqual( + task.result["message"], "No matching instances found" + ) # Because the instance IDs are from another account + + # Making sure that nothing changed in both accounts + self.assertIsNone(new_org_unit.location) + self.assertIsNone(self.org_unit_no_location.location) + self.assertEqual(self.org_unit_with_default_location.location, self.default_location) + + def test_overwrite_existing_location(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that overwrite existing org unit locations""" + # Setting a new location for both org_units + location = Point(42, 69, 420, 4326) + for org_unit in [self.org_unit_with_default_location, self.org_unit_with_other_location]: + org_unit.location = location + org_unit.save() + + # Linking both instances to these org_units + self.instance_with_default_location.org_unit = self.org_unit_with_default_location + self.instance_with_default_location.save() + self.instance_with_other_location.org_unit = self.org_unit_with_other_location + self.instance_with_other_location.save() + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_with_default_location.id, self.instance_with_other_location.id], + }, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be a success + self.runAndValidateTask(task, "SUCCESS") + + self.org_unit_with_default_location.refresh_from_db() + self.org_unit_with_other_location.refresh_from_db() + self.assertEqualLocations(self.org_unit_with_default_location.location, self.default_location) + self.assertEqualLocations(self.org_unit_with_other_location.location, self.other_location) + self.assertIsNone(self.org_unit_no_location.location) + + def test_no_location(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that don't have any location defined""" + # Let's create another instance without a location, but this time it's linked to self.org_unit_with_default_location + _ = m.Instance.objects.create( + form=self.form, + period="202001", + org_unit=self.org_unit_with_default_location, + project=self.project, + created_by=self.user, + export_id="noLoc", + ) + + self.client.force_authenticate(self.user) + # For a change, let's select everything, but remove the two instances with a location + # (= it's the same as electing both instances without location) + response = self.client.post( + self.BASE_URL, + data={ + "unselected_ids": [self.instance_with_default_location.id, self.instance_with_other_location.id], + }, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be a success + self.runAndValidateTask(task, "SUCCESS") + + self.org_unit_with_default_location.refresh_from_db() + self.org_unit_no_location.refresh_from_db() + self.assertIsNone(self.org_unit_with_default_location.location) # Got overwritten by None + self.assertIsNone(self.org_unit_no_location.location) # Still None + self.assertEqualLocations(self.org_unit_with_other_location.location, self.other_location) # Not updated + + def test_multiple_updates_same_org_unit(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that target the same orgunit""" + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_same_org_unit", result) + for instance in [ + self.instance_without_location, + self.instance_with_other_location, + self.instance_with_default_location, + ]: + self.assertIn(str(instance.id), result) + + def test_read_only_data_source(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that target orgunits which are part of a read-only data source""" + self.data_source.read_only = True + self.data_source.save() + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_read_only_source", result) + for instance in [ + self.instance_without_location, + self.instance_with_other_location, + self.instance_with_default_location, + ]: + self.assertIn(str(instance.id), result) + + def test_all_errors(self): + """POST /api/tasks/create/instancebulkgpspush/ all errors are triggered""" + # Preparing a new read-only data source + new_data_source = m.DataSource.objects.create(name="new data source", read_only=True) + new_version = m.SourceVersion.objects.create(data_source=new_data_source, number=2) + new_data_source.projects.set([self.project]) + new_org_unit = m.OrgUnit.objects.create( + name="new org unit", + org_unit_type=self.org_unit_type, + validation_status=m.OrgUnit.VALIDATION_VALID, + version=new_version, + source_ref="new org unit", + ) + new_instance = m.Instance.objects.create( + org_unit=new_org_unit, + form=self.form, + period="202004", + project=self.project, + created_by=self.user, + export_id="instance4", + location=self.default_location, + ) + + # Changing this org unit so that it does not trigger error_same_org_unit + self.instance_with_default_location.org_unit = self.org_unit_with_default_location + self.instance_with_default_location.save() + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_read_only_source", result) + self.assertIn("error_same_org_unit", result) + for instance in [self.instance_without_location, self.instance_with_other_location, new_instance]: + self.assertIn(str(instance.id), result) # Instead, we should probably check in which error they end up + self.assertNotIn(str(self.instance_with_default_location.id), result) + + def test_task_kill(self): + """Launch the task and then kill it + Note this actually doesn't work if it's killed while in the transaction part. + """ + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + data = self.assertJSONResponse(response, status.HTTP_201_CREATED) + self.assertValidTaskAndInDB(data["task"]) + + task = Task.objects.get(id=data["task"]["id"]) + task.should_be_killed = True + task.save() + + self.runAndValidateTask(task, "KILLED") + + def assertEqualLocations(self, point_1: Point, point_2: Point): + self.assertEqual(point_1.x, point_2.x) + self.assertEqual(point_1.y, point_2.y) + self.assertEqual(point_1.z, point_2.z) + self.assertEqual(point_1.srid, point_2.srid) diff --git a/iaso/urls.py b/iaso/urls.py index e0d023b409..a831b2fbc7 100644 --- a/iaso/urls.py +++ b/iaso/urls.py @@ -93,6 +93,7 @@ from .api.tasks import TaskSourceViewSet from .api.tasks.create.export_mobile_setup import ExportMobileSetupViewSet from .api.tasks.create.import_gpkg import ImportGPKGViewSet +from .api.tasks.create.instance_bulk_gps_push import InstanceBulkGpsPushViewSet from .api.tasks.create.org_unit_bulk_location_set import OrgUnitsBulkLocationSet from .api.user_roles import UserRolesViewSet from .api.workflows.changes import WorkflowChangeViewSet @@ -168,6 +169,7 @@ router.register(r"tasks/create/orgunitsbulklocationset", OrgUnitsBulkLocationSet, basename="orgunitsbulklocationset") router.register(r"tasks/create/importgpkg", ImportGPKGViewSet, basename="importgpkg") router.register(r"tasks/create/exportmobilesetup", ExportMobileSetupViewSet, basename="exportmobilesetup") +router.register(r"tasks/create/instancebulkgpspush", InstanceBulkGpsPushViewSet, basename="instancebulkgpspush") router.register(r"tasks", TaskSourceViewSet, basename="tasks") router.register(r"comments", CommentViewSet, basename="comments") router.register(r"entities", EntityViewSet, basename="entity") diff --git a/iaso/utils/models/common.py b/iaso/utils/models/common.py index 77dd3c031c..541ee3b53a 100644 --- a/iaso/utils/models/common.py +++ b/iaso/utils/models/common.py @@ -1,3 +1,7 @@ +from typing import Dict, List + +from django.db.models import QuerySet + from iaso.models.base import User @@ -23,3 +27,76 @@ def get_org_unit_parents_ref(field_name, org_unit, parent_source_ref_field_names if parent_ref: return f"iaso#{parent_ref}" return None + + +def check_instance_bulk_gps_push(queryset: QuerySet) -> (bool, Dict[str, List[int]], Dict[str, List[int]]): + """ + Determines if there are any warnings or errors if the given Instances were to push their own location to their OrgUnit. + + There are 2 types of warnings: + - warning_no_location: if an Instance doesn't have any location + - warning_overwrite: if the Instance's OrgUnit already has a location + The gps push can be performed even if there are any warnings, keeping in mind the consequences. + + There are 2 types of errors: + - error_same_org_unit: if there are multiple Instances in the given queryset that share the same OrgUnit + - error_read_only_source: if any Instance's OrgUnit is part of a read-only DataSource + The gps push cannot be performed if there are any errors. + """ + # Variables used for warnings + set_org_units_ids = set() + overwrite_ids = [] + no_location_ids = [] + + # Variables used for errors + org_units_to_instances_dict = {} + read_only_data_sources = [] + + for instance in queryset: + # First, let's check for potential errors + org_unit = instance.org_unit + if org_unit.id in org_units_to_instances_dict: + # we can't push this instance's location since there was another instance linked to this OrgUnit + org_units_to_instances_dict[org_unit.id].append(instance.id) + continue + else: + org_units_to_instances_dict[org_unit.id] = [instance.id] + + if org_unit.version and org_unit.version.data_source.read_only: + read_only_data_sources.append(instance.id) + continue + + # Then, let's check for potential warnings + if not instance.location: + no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit + continue + + set_org_units_ids.add(org_unit.id) + if org_unit.location or org_unit.geom: + overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location + continue + + # Before returning, we need to check if we've had multiple hits on an OrgUnit + error_same_org_unit_ids = _check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + + success: bool = not read_only_data_sources and not error_same_org_unit_ids + errors = {} + if read_only_data_sources: + errors["error_read_only_source"] = read_only_data_sources + if error_same_org_unit_ids: + errors["error_same_org_unit"] = error_same_org_unit_ids + warnings = {} + if no_location_ids: + warnings["warning_no_location"] = no_location_ids + if overwrite_ids: + warnings["warning_overwrite"] = overwrite_ids + + return success, errors, warnings + + +def _check_bulk_gps_repeated_org_units(org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: + error_instance_ids = [] + for _, instance_ids in org_units_to_instance_ids.items(): + if len(instance_ids) >= 2: + error_instance_ids.extend(instance_ids) + return error_instance_ids From 0c64cbdb8da2965c68ce1b3ab8531502732e6591 Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Fri, 20 Dec 2024 15:29:58 +0200 Subject: [PATCH 11/62] launch task --- .../components/PushGpsDialogComponent.tsx | 53 ++++++++++++++----- .../hooks/useInstanceBulkgpspush.tsx | 23 ++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 38ec59fb04..e54e884d0b 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -1,12 +1,14 @@ -import React, { FunctionComponent, useState } from 'react'; +import React, { FunctionComponent, useCallback, useState } from 'react'; import { Grid, Typography } from '@mui/material'; -import { useSafeIntl } from 'bluesquare-components'; +import { useRedirectTo, useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; import { Instance } from '../types/instance'; import { Selection } from '../../orgUnits/types/selection'; import { useGetCheckBulkGpsPush } from '../hooks/useGetCheckBulkGpsPush'; import PushBulkGpsWarning from './PushBulkGpsWarning'; +import { useInstanceBulkgpspush } from '../hooks/useInstanceBulkgpspush'; +import { baseUrls } from '../../../constants/urls'; type Props = { renderTrigger: (openDialog: boolean) => void; @@ -21,16 +23,41 @@ const PushGpsDialogComponent: FunctionComponent = ({ useState(false); const [approveSubmissionNoHasGps, setApproveSubmissionNoHasGps] = useState(false); - const onConfirm = closeDialog => { - return null; - }; + const { mutateAsync: bulkgpspush } = useInstanceBulkgpspush(); + const select_all = selection.selectAll; + const selected_ids = selection.selectedItems; + const unselected_ids = selection.unSelectedItems; + const instancebulkgpspush = useCallback(async () => { + await bulkgpspush({ + select_all, + selected_ids: selected_ids.map(item => item.id), + unselected_ids: unselected_ids.map(item => item.id), + }); + }, [bulkgpspush, select_all, selected_ids, unselected_ids]); + + const onConfirm = useCallback( + async closeDialog => { + await instancebulkgpspush(); + closeDialog(); + }, + [instancebulkgpspush], + ); + const redirectTo = useRedirectTo(); + const onConfirmAndSeeTask = useCallback( + async closeDialog => { + await instancebulkgpspush(); + closeDialog(); + redirectTo(baseUrls.tasks, { + order: '-created_at', + }); + }, + [instancebulkgpspush, redirectTo], + ); const { data: checkBulkGpsPush, isError } = useGetCheckBulkGpsPush({ - selected_ids: selection.selectedItems.map(item => item.id).join(','), - select_all: selection.selectAll, - unselected_ids: selection.unSelectedItems - .map(item => item.id) - .join(','), + selected_ids: selected_ids.map(item => item.id).join(','), + select_all, + unselected_ids: unselected_ids.map(item => item.id).join(','), }); const onClosed = () => { @@ -66,14 +93,16 @@ const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger(openDialog)} titleMessage={title} - onConfirm={onConfirm} + onConfirm={closeDialog => onConfirm(closeDialog)} confirmMessage={MESSAGES.launch} onClosed={onClosed} cancelMessage={MESSAGES.cancel} maxWidth="sm" additionalButton additionalMessage={MESSAGES.goToCurrentTask} - onAdditionalButtonClick={onConfirm} + onAdditionalButtonClick={closeDialog => + onConfirmAndSeeTask(closeDialog) + } allowConfimAdditionalButton > diff --git a/hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx b/hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx new file mode 100644 index 0000000000..c8c4d9123d --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx @@ -0,0 +1,23 @@ +import { UseMutationResult } from 'react-query'; +import { postRequest } from '../../../libs/Api'; +import { useSnackMutation } from '../../../libs/apiHooks'; + +type BulkGpsPush = { + select_all: boolean; + selected_ids: string[]; + unselected_ids: string[]; +}; +export const useInstanceBulkgpspush = (): UseMutationResult => { + return useSnackMutation({ + mutationFn: (data: BulkGpsPush) => { + const { select_all, selected_ids, unselected_ids } = data; + + return postRequest('/api/tasks/create/instancebulkgpspush/', { + select_all, + selected_ids, + unselected_ids, + }); + }, + showSucessSnackBar: false, + }); +}; From 2d0e1a741708e521cd2717ee48bd91c1b44841ba Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Fri, 20 Dec 2024 16:09:33 +0200 Subject: [PATCH 12/62] remove see all link --- .../components/PushBulkGpsWarning.tsx | 8 +-- .../components/PushGpsDialogComponent.tsx | 57 +++++++++++++++---- .../hooks/useGetCheckBulkGpsPush.tsx | 3 +- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx index 53114365ea..eefcc47c00 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx @@ -1,19 +1,17 @@ import { Button, Grid, Typography } from '@mui/material'; -import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import { useSafeIntl } from 'bluesquare-components'; import React, { FunctionComponent } from 'react'; import MESSAGES from '../messages'; type Props = { condition: boolean; message: { defaultMessage: string; id: string }; - linkTo: string; approveCondition: boolean; onApproveClick: () => void; }; const PushBulkGpsWarning: FunctionComponent = ({ condition, message, - linkTo, approveCondition, onApproveClick, }) => { @@ -36,11 +34,11 @@ const PushBulkGpsWarning: FunctionComponent = ({ - + {/* {formatMessage(MESSAGES.seeAll)} - + */} = ({ alignItems="center" direction="row" > - - + + - Some OrgUnits already have GPS coordinates. Do - you want to proceed and overwrite them? + {formatMessage( + MESSAGES.someOrgUnitsHasAlreadyGps, + )} - - - - See all - - + + + {formatMessage(MESSAGES.seeAll)} + - - - - See all - - + + diff --git a/hat/assets/js/apps/Iaso/domains/instances/messages.js b/hat/assets/js/apps/Iaso/domains/instances/messages.js index addfe1b8a4..ef954b5766 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/messages.js +++ b/hat/assets/js/apps/Iaso/domains/instances/messages.js @@ -211,7 +211,24 @@ const MESSAGES = defineMessages({ 'You are about to push the locations of {submissionCount} instances to {orgUnitCount} OrgUnits', id: 'iaso.instance.pushGpsWarningMessage', }, - + noGpsForSomeInstaces: { + defaultMessage: + "Some instances don't have locations. Nothing will be applied for those OrgUnits.", + id: 'iaso.instance.noGpsForSomeInstaces', + }, + someOrgUnitsHasAlreadyGps: { + defaultMessage: + 'Some OrgUnits already have GPS coordinates. Do you want to proceed and overwrite them?', + id: 'iaso.instance.someOrgUnitsHasAlreadyGps', + }, + seeAll: { + defaultMessage: 'See all', + id: 'iaso.label.seeAll', + }, + approve: { + defaultMessage: 'Approve', + id: 'iaso.label.approve', + }, launch: { id: 'iaso.label.launch', defaultMessage: 'Launch', From a8db0301462e73b3c9685e0dc3ad20111f29b391 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 13 Dec 2024 19:05:50 +0100 Subject: [PATCH 37/62] API call check push gps - WIP --- iaso/api/instances.py | 109 +++++++++++++++++++- iaso/tests/api/test_instances.py | 170 +++++++++++++++++++++++++++++-- 2 files changed, 271 insertions(+), 8 deletions(-) diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 8e1b09f548..8f87b5a8ea 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -3,7 +3,7 @@ import ntpath from copy import copy from time import gmtime, strftime -from typing import Any, Dict, Union +from typing import Any, Dict, Union, List import pandas as pd from django.contrib.auth.models import User @@ -43,7 +43,14 @@ from ..utils.models.common import get_creator_name from . import common from .comment import UserSerializerForComment -from .common import CONTENT_TYPE_CSV, CONTENT_TYPE_XLSX, FileFormatEnum, TimestampField, safe_api_import +from .common import ( + CONTENT_TYPE_CSV, + CONTENT_TYPE_XLSX, + FileFormatEnum, + TimestampField, + safe_api_import, + parse_comma_separated_numeric_values, +) from .instance_filters import get_form_from_instance_filters, parse_instance_filters logger = logging.getLogger(__name__) @@ -598,6 +605,104 @@ def bulkdelete(self, request): status=201, ) + @action(detail=False, methods=["GET"], permission_classes=[permissions.IsAuthenticated, HasInstancePermission]) + def check_bulk_gps_push(self, request): + # first, let's parse all parameters received from the URL + select_all, selected_ids, unselected_ids = self._parse_check_bulk_gps_push_parameters(request.GET) + + # then, let's make sure that each ID actually exists and that the user has access to it + instances_query = self.get_queryset() + for selected_id in selected_ids: + get_object_or_404(instances_query, pk=selected_id) + for unselected_id in unselected_ids: + get_object_or_404(instances_query, pk=unselected_id) + + # let's filter everything + filters = parse_instance_filters(request.GET) + instances_query = instances_query.select_related("org_unit") + instances_query = instances_query.exclude(file="").exclude(device__test_device=True) + instances_query = instances_query.for_filters(**filters) + + if not select_all: + instances_query = instances_query.filter(pk__in=selected_ids) + else: + instances_query = instances_query.exclude(pk__in=unselected_ids) + + overwrite_ids = [] + no_location_ids = [] + org_units_to_instances_dict = {} + set_org_units_ids = set() + + for instance in instances_query: + if not instance.location: + no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit + continue + + org_unit = instance.org_unit + if org_unit.id in org_units_to_instances_dict: + # we can't push this instance's location since there was another instance linked to this OrgUnit + org_units_to_instances_dict[org_unit.id].append(instance.id) + continue + else: + org_units_to_instances_dict[org_unit.id] = [instance.id] + + set_org_units_ids.add(org_unit.id) + if org_unit.location or org_unit.geom: + overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location + continue + + # Before returning, we need to check if we've had multiple hits on an OrgUnit + error_same_org_unit_ids = self._check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + + if len(error_same_org_unit_ids): + return Response( + {"result": "error", "error_ids": error_same_org_unit_ids}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if len(no_location_ids) or len(overwrite_ids): + dict_response = { + "result": "warnings", + } + if len(no_location_ids): + dict_response["warning_no_location"] = no_location_ids + if len(overwrite_ids): + dict_response["warning_overwrite"] = overwrite_ids + + return Response(dict_response, status=status.HTTP_200_OK) + + return Response( + { + "result": "success", + }, + status=status.HTTP_200_OK, + ) + + def _parse_check_bulk_gps_push_parameters(self, query_parameters): + raw_select_all = query_parameters.get("select_all", True) + select_all = raw_select_all not in ["false", "False", "0"] + + raw_selected_ids = query_parameters.get("selected_ids", None) + if raw_selected_ids: + selected_ids = parse_comma_separated_numeric_values(raw_selected_ids, "selected_ids") + else: + selected_ids = [] + + raw_unselected_ids = query_parameters.get("unselected_ids", None) + if raw_unselected_ids: + unselected_ids = parse_comma_separated_numeric_values(raw_unselected_ids, "unselected_ids") + else: + unselected_ids = [] + + return select_all, selected_ids, unselected_ids + + def _check_bulk_gps_repeated_org_units(self, org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: + error_instance_ids = [] + for _, instance_ids in org_units_to_instance_ids.items(): + if len(instance_ids) >= 2: + error_instance_ids.extend(instance_ids) + return error_instance_ids + QUERY = """ select DATE_TRUNC('month', COALESCE(iaso_instance.source_created_at, iaso_instance.created_at)) as month, (select name from iaso_form where id = iaso_instance.form_id) as form_name, diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 5675da1859..7bea3775a7 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -1925,16 +1925,174 @@ def test_instance_retrieve_with_related_change_requests(self): # Test instance with no change request response = self.client.get(f"/api/instances/{instance_reference.id}/") self.assertEqual(response.status_code, 200) - respons_json = response.json() - self.assertListEqual(respons_json["change_requests"], []) + response_json = response.json() + self.assertListEqual(response_json["change_requests"], []) m.OrgUnitChangeRequest.objects.create(org_unit=org_unit, new_name="Modified org unit") response = self.client.get(f"/api/instances/{instance_reference.id}/") - respons_json = response.json() - self.assertListEqual(respons_json["change_requests"], []) + response_json = response.json() + self.assertListEqual(response_json["change_requests"], []) # Test instance with change request linked to it change_request_instance_reference = m.OrgUnitChangeRequest.objects.create(org_unit=org_unit) change_request_instance_reference.new_reference_instances.add(instance_reference) response = self.client.get(f"/api/instances/{instance_reference.id}/") - respons_json = response.json() - self.assertEqual(respons_json["change_requests"][0]["id"], change_request_instance_reference.id) + response_json = response.json() + self.assertEqual(response_json["change_requests"][0]["id"], change_request_instance_reference.id) + + def test_check_bulk_push_gps_select_all_ok(self): + # pushing gps data means that we need a mapping of 1 instance to 1 orgunit + for instance in [self.instance_2, self.instance_3, self.instance_4]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + instance.refresh_from_db() + + # setting gps data for instances that were not deleted + self.instance_5.org_unit = self.jedi_council_endor + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + new_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_5, self.instance_6, self.instance_8]: + instance.location = new_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + self.assertEqual(response_json["result"], "success") + + def test_check_bulk_push_gps_select_all_error(self): + # setting gps data for instances that were not deleted + self.instance_1.org_unit = self.jedi_council_endor + self.instance_2.org_unit = self.jedi_council_endor + new_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_2]: + instance.location = new_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "error") + self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + + def test_check_bulk_push_gps_select_all_warning_no_location(self): + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # Since all instances in the setup don't have location data, they will show up in warnings + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual( + response_json["warning_no_location"], m.Instance.non_deleted_objects.values_list("id", flat=True) + ) + + def test_check_bulk_push_gps_select_all_warning_overwrite(self): + # pushing gps data means that we need a mapping of 1 instance to 1 orgunit + for instance in [self.instance_2, self.instance_3, self.instance_4]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + instance.refresh_from_db() + + # no org unit was assigned, so we select some + self.instance_5.org_unit = self.jedi_council_endor + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + new_org_unit_location = Point(1, 2, 3) + new_instance_location = Point(4, 5, 6) + non_deleted_instances = [self.instance_1, self.instance_5, self.instance_6, self.instance_8] + for instance in non_deleted_instances: + # setting gps data for org_units whose instance was not deleted + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + # setting gps data for these instances as well, since we want to override the org_unit location + instance.location = new_instance_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [i.id for i in non_deleted_instances]) + + def test_check_bulk_push_gps_selected_ids_ok(self): + self.client.force_authenticate(self.yoda) + new_instance = self.create_form_instance( + form=self.form_1, + period="2024Q4", + org_unit=self.jedi_council_corruscant, + project=self.project, + created_by=self.yoda, + location=Point(5, 6.45, 2.33), + ) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={new_instance.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # Instance has a location, but OrgUnit doesn't, so check is ok, location could be pushed + self.assertEqual(response_json["result"], "success") + + def test_check_bulk_push_gps_selected_ids_error(self): + self.client.force_authenticate(self.yoda) + # Setting GPS data for these 3 instances (same OrgUnit) + for instance in [self.instance_1, self.instance_2, self.instance_3]: + instance.location = Point(5, 6.45, 2.33) + instance.save() + self.instance_1.refresh_from_db() + self.instance_2.refresh_from_db() + self.instance_3.refresh_from_db() + + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id},{self.instance_3.id}" + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "error") + self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + + def test_check_bulk_push_gps_selected_ids_warning_no_location(self): + self.client.force_authenticate(self.yoda) + # Linking these instances to some orgunits + self.instance_1.org_unit = self.jedi_council_corruscant + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + for instance in [self.instance_1, self.instance_2, self.instance_3]: + instance.save() + instance.refresh_from_db() + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id},{self.instance_3.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual( + response_json["warning_no_location"], [self.instance_1.id, self.instance_2.id, self.instance_3.id] + ) + + def test_check_bulk_push_gps_selected_ids_warning_overwrite(self): + pass + + def test_check_bulk_push_gps_unselected_ids_ok(self): + pass + + def test_check_bulk_push_gps_unselected_ids_error(self): + pass + + def test_check_bulk_push_gps_unselected_ids_warning_no_location(self): + pass + + def test_check_bulk_push_gps_unselected_ids_warning_overwrite(self): + pass From 85f7c70d5baa8696f5ef6e84ff01915c55765a60 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 13 Dec 2024 20:34:49 +0100 Subject: [PATCH 38/62] Added tests with wrong account and wrong IDs --- iaso/tests/api/test_instances.py | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 7bea3775a7..7ac3e697f1 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -2062,6 +2062,44 @@ def test_check_bulk_push_gps_selected_ids_error(self): self.assertEqual(response_json["result"], "error") self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): + self.client.force_authenticate(self.yoda) + probably_not_a_valid_id = 1234567980 + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={probably_not_a_valid_id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + + def test_check_bulk_push_gps_selected_ids_error_wrong_account(self): + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) + new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + new_instance = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + ) + + self.client.force_authenticate(self.yoda) + # Checking instance from new account + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={new_instance.id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + def test_check_bulk_push_gps_selected_ids_warning_no_location(self): self.client.force_authenticate(self.yoda) # Linking these instances to some orgunits @@ -2091,6 +2129,44 @@ def test_check_bulk_push_gps_unselected_ids_ok(self): def test_check_bulk_push_gps_unselected_ids_error(self): pass + def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): + self.client.force_authenticate(self.yoda) + probably_not_a_valid_id = 1234567980 + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&unselected_ids={probably_not_a_valid_id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + + def test_check_bulk_push_gps_unselected_ids_error_wrong_account(self): + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) + new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + new_instance = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + ) + + self.client.force_authenticate(self.yoda) + # Checking instance from new account + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&unselected_ids={new_instance.id}" + ) + self.assertContains( + response, + f"Not found", + status_code=status.HTTP_404_NOT_FOUND, + ) + def test_check_bulk_push_gps_unselected_ids_warning_no_location(self): pass From 3a4381b33e4cbae53fa2405545d4d761ac61cc56 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Sat, 14 Dec 2024 00:20:40 +0100 Subject: [PATCH 39/62] Added missing tests --- iaso/tests/api/test_instances.py | 232 +++++++++++++++++++++++++++++-- 1 file changed, 220 insertions(+), 12 deletions(-) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index 7ac3e697f1..e7defdfddc 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -2025,6 +2025,44 @@ def test_check_bulk_push_gps_select_all_warning_overwrite(self): self.assertEqual(response_json["result"], "warnings") self.assertCountEqual(response_json["warning_overwrite"], [i.id for i in non_deleted_instances]) + def test_check_bulk_push_gps_select_all_warning_both(self): + # pushing gps data means that we need a mapping of 1 instance to 1 orgunit + for instance in [self.instance_2, self.instance_3, self.instance_4]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + instance.refresh_from_db() + + # instances 1 and 5 will overwrite their org_unit location + new_instance_location = Point(4, 5, 6) + self.instance_5.org_unit = self.jedi_council_endor + self.instance_5.location = new_instance_location + self.instance_1.location = new_instance_location + + # instances 6 and 8 will have no location data + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + + new_org_unit_location = Point(1, 2, 3) + non_deleted_instances = [self.instance_1, self.instance_5, self.instance_6, self.instance_8] + for instance in non_deleted_instances: + # setting gps data for org_units whose instance was not deleted + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_5.id]) + self.assertCountEqual(response_json["warning_no_location"], [self.instance_6.id, self.instance_8.id]) + def test_check_bulk_push_gps_selected_ids_ok(self): self.client.force_authenticate(self.yoda) new_instance = self.create_form_instance( @@ -2049,9 +2087,7 @@ def test_check_bulk_push_gps_selected_ids_error(self): for instance in [self.instance_1, self.instance_2, self.instance_3]: instance.location = Point(5, 6.45, 2.33) instance.save() - self.instance_1.refresh_from_db() - self.instance_2.refresh_from_db() - self.instance_3.refresh_from_db() + instance.refresh_from_db() response = self.client.get( f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id},{self.instance_3.id}" @@ -2076,9 +2112,13 @@ def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): def test_check_bulk_push_gps_selected_ids_error_wrong_account(self): # Preparing new setup - new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) - new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) new_instance = self.create_form_instance( form=new_form, @@ -2121,13 +2161,101 @@ def test_check_bulk_push_gps_selected_ids_warning_no_location(self): ) def test_check_bulk_push_gps_selected_ids_warning_overwrite(self): - pass + # no org unit was assigned, so we select some + self.instance_1.org_unit = self.jedi_council_endor + self.instance_2.org_unit = self.jedi_council_corruscant + new_org_unit_location = Point(1, 2, 3) + new_instance_location = Point(4, 5, 6) + for instance in [self.instance_1, self.instance_2]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + # setting gps data for these instances as well, since we want to override the org_unit location + instance.location = new_instance_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id},{self.instance_2.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_2.id]) + + def test_check_bulk_push_gps_selected_ids_warning_both(self): + # instances 1 and 5 will overwrite their org_unit location + new_instance_location = Point(4, 5, 6) + self.instance_5.org_unit = self.jedi_council_endor + self.instance_5.location = new_instance_location + self.instance_1.location = new_instance_location + + # instances 6 and 8 will have no location data + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + + new_org_unit_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_5, self.instance_6, self.instance_8]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?select_all=false&selected_ids={self.instance_1.id}," + f"{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_5.id]) + self.assertCountEqual(response_json["warning_no_location"], [self.instance_6.id, self.instance_8.id]) def test_check_bulk_push_gps_unselected_ids_ok(self): - pass + self.instance_1.location = Point(1, 2, 3) + self.instance_1.save() + self.instance_1.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_1 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_2.id},{self.instance_3.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "success") def test_check_bulk_push_gps_unselected_ids_error(self): - pass + # no org unit was assigned, so we select some + for instance in [self.instance_1, self.instance_2]: + instance.org_unit = self.jedi_council_endor + instance.location = Point(1, 2, 3) + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_1 & instance_2 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_3.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "error") + self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) @@ -2143,9 +2271,13 @@ def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): def test_check_bulk_push_gps_unselected_ids_error_wrong_account(self): # Preparing new setup - new_account, new_data_source, _, new_project = self.create_account_datasource_version_project("new source", "new account", "new project") + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) new_user, _, _ = self.create_base_users(new_account, ["iaso_submissions"]) - new_org_unit = m.OrgUnit.objects.create(name="New Org Unit", source_ref="new org unit", validation_status="VALID") + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) new_instance = self.create_form_instance( form=new_form, @@ -2168,7 +2300,83 @@ def test_check_bulk_push_gps_unselected_ids_error_wrong_account(self): ) def test_check_bulk_push_gps_unselected_ids_warning_no_location(self): - pass + # Changing instance_1 org unit in order to be different from instance_2 + self.instance_1.org_unit = self.jedi_council_endor + self.instance_1.save() + self.instance_1.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_1 & instance_2 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_3.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_no_location"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_warning_overwrite(self): - pass + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + + # no org unit was assigned, so we select some + new_org_unit_location = Point(1, 2, 3) + new_instance_location = Point(4, 5, 6) + for instance in [self.instance_2, self.instance_3]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + # setting gps data for these instances as well, since we want to override the org_unit location + instance.location = new_instance_location + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + # only pushing instance_3 & instance_2 GPS + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_1.id}," + f"{self.instance_4.id},{self.instance_5.id},{self.instance_6.id},{self.instance_8.id}," + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_3.id, self.instance_2.id]) + + def test_check_bulk_push_gps_unselected_ids_warning_both(self): + # instances 1 and 5 will overwrite their org_unit location + new_instance_location = Point(4, 5, 6) + self.instance_5.org_unit = self.jedi_council_endor + self.instance_5.location = new_instance_location + self.instance_1.location = new_instance_location + + # instances 6 and 8 will have no location data + self.instance_6.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + + new_org_unit_location = Point(1, 2, 3) + for instance in [self.instance_1, self.instance_5, self.instance_6, self.instance_8]: + # setting gps data for org_units + org_unit = instance.org_unit + org_unit.location = new_org_unit_location + org_unit.save() + org_unit.refresh_from_db() + + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get( + f"/api/instances/check_bulk_gps_push/?unselected_ids={self.instance_2.id},{self.instance_3.id},{self.instance_4.id}" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response_json = response.json() + # All these Instances target the same OrgUnit, so it's impossible to push gps data + self.assertEqual(response_json["result"], "warnings") + self.assertCountEqual(response_json["warning_overwrite"], [self.instance_1.id, self.instance_5.id]) + self.assertCountEqual(response_json["warning_no_location"], [self.instance_6.id, self.instance_8.id]) From 7fc32f9cc6ca32402da166cc277fc1f09c0c7e9b Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Tue, 17 Dec 2024 15:23:10 +0200 Subject: [PATCH 40/62] get warnings and status from backend --- .../components/PushGpsDialogComponent.tsx | 200 ++++++++++-------- .../hooks/useGetCheckBulkGpsPush.tsx | 28 +++ .../Iaso/domains/instances/types/instance.ts | 5 + .../Iaso/domains/instances/utils/index.tsx | 1 - 4 files changed, 145 insertions(+), 89 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 9029357b64..58fb55b691 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -3,33 +3,39 @@ import { Button, Grid, Typography } from '@mui/material'; import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; -import { createExportRequest } from '../actions'; import { Instance } from '../types/instance'; import { Selection } from '../../orgUnits/types/selection'; +import { useGetCheckBulkGpsPush } from '../hooks/useGetCheckBulkGpsPush'; type Props = { - getFilters: () => void; renderTrigger: (openDialog: boolean) => void; selection: Selection; }; const PushGpsDialogComponent: FunctionComponent = ({ - getFilters, renderTrigger, selection, }) => { - const [forceExport, setForceExport] = useState(false); const [approveOrgUnitHasGps, setApproveOrgUnitHasGps] = useState(false); const [approveSubmissionNoHasGps, setApproveSubmissionNoHasGps] = useState(false); const onConfirm = closeDialog => { - const filterParams = getFilters(); - createExportRequest({ forceExport, ...filterParams }, selection).then( - () => closeDialog(), - ); + return null; }; + + const { data: checkBulkGpsPush, isFetching: isLoadingCheckBulkGpsPush } = + useGetCheckBulkGpsPush({ + selected_ids: selection.selectedItems + .map(item => item.id) + .join(','), + select_all: selection.selectAll, + unselected_ids: selection.unSelectedItems + .map(item => item.id) + .join(','), + }); + const onClosed = () => { - setForceExport(false); + return null; }; const onApprove = type => { @@ -83,98 +89,116 @@ const PushGpsDialogComponent: FunctionComponent = ({ })} - - - - - {formatMessage(MESSAGES.noGpsForSomeInstaces)} - - - + {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && ( - - {formatMessage(MESSAGES.seeAll)} - - - - - - - - - + {formatMessage(MESSAGES.seeAll)} + + + - + + + )} + {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && ( - - {formatMessage(MESSAGES.seeAll)} - - - - + + {formatMessage(MESSAGES.seeAll)} + + + + + - + )} ); diff --git a/hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx b/hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx new file mode 100644 index 0000000000..3783fab996 --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/hooks/useGetCheckBulkGpsPush.tsx @@ -0,0 +1,28 @@ +import { UseQueryResult } from 'react-query'; +import { useSnackQuery } from '../../../libs/apiHooks'; +import MESSAGES from '../messages'; +import { getRequest } from '../../../libs/Api'; +import { makeUrlWithParams } from '../../../libs/utils'; +import { CheckBulkGpsPushResult } from '../types/instance'; + +export const useGetCheckBulkGpsPush = ( + params, +): UseQueryResult => { + return useSnackQuery({ + queryKey: ['bulkGpsCheck', params], + queryFn: () => { + return getRequest( + makeUrlWithParams( + '/api/instances/check_bulk_gps_push/', + params, + ), + ); + }, + snackErrorMsg: MESSAGES.error, + options: { + select: data => { + return data; + }, + }, + }); +}; diff --git a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts index dc06113091..af1bcaca0f 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts @@ -104,6 +104,11 @@ export interface PaginatedInstances extends Pagination { instances: Instance[]; } +export type CheckBulkGpsPushResult = { + result: string; + warning_no_location?: number[]; + warning_overwrite?: number[]; +}; export type MimeType = // Text | 'text/plain' diff --git a/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx b/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx index 8be7cda7c5..96b3082d0a 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/utils/index.tsx @@ -502,7 +502,6 @@ export const useSelectionActions = ( filters} renderTrigger={openDialog => { const iconDisabled = newSelection.selectCount === 0; const iconProps = { From bbb01cfbbb64ae5eff2b00d1ca74cc02c25cc5fd Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Tue, 17 Dec 2024 18:09:25 +0200 Subject: [PATCH 41/62] WIP --- .../Iaso/domains/app/translations/en.json | 1 + .../Iaso/domains/app/translations/fr.json | 1 + .../components/PushGpsDialogComponent.tsx | 210 +++++++++--------- .../apps/Iaso/domains/instances/messages.js | 4 + .../Iaso/domains/instances/types/instance.ts | 1 + 5 files changed, 116 insertions(+), 101 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index c5701b4a48..d6418ab956 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -384,6 +384,7 @@ "iaso.instance.logs.versionB": "Version B", "iaso.instance.missingFile": "Cannot find an instance with a file", "iaso.instance.missingGeolocation": "Cannot find an instance with geolocation", + "iaso.instance.multipleInstancesOneOrgUnitWarningMessage": "Multiple submissions are using the same org unit", "iaso.instance.noGpsForSomeInstaces": "Some instances don't have locations. Nothing will be applied for those OrgUnits.", "iaso.instance.NoLocksHistory": "There's no history", "iaso.instance.org_unit": "Org unit", diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index 07a5b66116..8f5f1e5f55 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -385,6 +385,7 @@ "iaso.instance.logs.versionB": "Version B", "iaso.instance.missingFile": "Aucun fichier trouvé", "iaso.instance.missingGeolocation": "Aucune soumission trouvée avec une localisation", + "iaso.instance.multipleInstancesOneOrgUnitWarningMessage": "Plusieurs soumissions utilisent la même unité d'org.", "iaso.instance.noGpsForSomeInstaces": "Certaines soumissions n'ont pas de GPS. Rien ne sera appliqué pour ces unités d'organisation.", "iaso.instance.NoLocksHistory": "Il n'y a pas d'historique", "iaso.instance.org_unit": "Unités d'organisation", diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 58fb55b691..43a93ba9ab 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -83,122 +83,130 @@ const PushGpsDialogComponent: FunctionComponent = ({ - {formatMessage(MESSAGES.pushGpsWarningMessage, { - submissionCount: 5, - orgUnitCount: 10, - })} + {(checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 + ? formatMessage(MESSAGES.pushGpsWarningMessage, { + submissionCount: selection.selectCount, + orgUnitCount: selection.selectCount, + }) + : formatMessage( + MESSAGES.multipleInstancesOneOrgUnitWarningMessage, + )} - {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && ( - - - - - {formatMessage( - MESSAGES.noGpsForSomeInstaces, - )} - - - + {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && + (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - - {formatMessage(MESSAGES.seeAll)} - - - - - - - )} - {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && ( - - - + {formatMessage(MESSAGES.seeAll)} + + + - + + + )} + {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && + (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - + + - - )} + )} ); diff --git a/hat/assets/js/apps/Iaso/domains/instances/messages.js b/hat/assets/js/apps/Iaso/domains/instances/messages.js index ef954b5766..a1d8be39cc 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/messages.js +++ b/hat/assets/js/apps/Iaso/domains/instances/messages.js @@ -221,6 +221,10 @@ const MESSAGES = defineMessages({ 'Some OrgUnits already have GPS coordinates. Do you want to proceed and overwrite them?', id: 'iaso.instance.someOrgUnitsHasAlreadyGps', }, + multipleInstancesOneOrgUnitWarningMessage: { + defaultMessage: 'Multiple submissions are using the same org unit', + id: 'iaso.instance.multipleInstancesOneOrgUnitWarningMessage', + }, seeAll: { defaultMessage: 'See all', id: 'iaso.label.seeAll', diff --git a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts index af1bcaca0f..996a34a3e1 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/types/instance.ts @@ -108,6 +108,7 @@ export type CheckBulkGpsPushResult = { result: string; warning_no_location?: number[]; warning_overwrite?: number[]; + error_ids?: number[]; }; export type MimeType = // Text From dcabf199b13d2e59bbe841ac61d250f9cd570ae4 Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Thu, 19 Dec 2024 10:51:42 +0200 Subject: [PATCH 42/62] refactor the code --- .../components/PushGpsDialogComponent.tsx | 214 +++++++----------- 1 file changed, 81 insertions(+), 133 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 43a93ba9ab..cde93c7737 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -11,6 +11,54 @@ type Props = { renderTrigger: (openDialog: boolean) => void; selection: Selection; }; + +const WarningSection = ({ + condition, + message, + linkTo, + approveCondition, + onApproveClick, +}) => { + const { formatMessage } = useSafeIntl(); + if (!condition) return null; + + return ( + + + + + {formatMessage(message)} + + + + + + {formatMessage(MESSAGES.seeAll)} + + + + + + + ); +}; + const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger, selection, @@ -23,16 +71,13 @@ const PushGpsDialogComponent: FunctionComponent = ({ return null; }; - const { data: checkBulkGpsPush, isFetching: isLoadingCheckBulkGpsPush } = - useGetCheckBulkGpsPush({ - selected_ids: selection.selectedItems - .map(item => item.id) - .join(','), - select_all: selection.selectAll, - unselected_ids: selection.unSelectedItems - .map(item => item.id) - .join(','), - }); + const { data: checkBulkGpsPush, isError } = useGetCheckBulkGpsPush({ + selected_ids: selection.selectedItems.map(item => item.id).join(','), + select_all: selection.selectAll, + unselected_ids: selection.unSelectedItems + .map(item => item.id) + .join(','), + }); const onClosed = () => { return null; @@ -59,9 +104,6 @@ const PushGpsDialogComponent: FunctionComponent = ({ if (selection) { title = { ...MESSAGES.pushGpsToOrgUnits, - values: { - count: selection.selectCount, - }, }; } const { formatMessage } = useSafeIntl(); @@ -83,130 +125,36 @@ const PushGpsDialogComponent: FunctionComponent = ({ - {(checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 - ? formatMessage(MESSAGES.pushGpsWarningMessage, { + {isError + ? formatMessage( + MESSAGES.multipleInstancesOneOrgUnitWarningMessage, + ) + : formatMessage(MESSAGES.pushGpsWarningMessage, { submissionCount: selection.selectCount, orgUnitCount: selection.selectCount, - }) - : formatMessage( - MESSAGES.multipleInstancesOneOrgUnitWarningMessage, - )} + })} - {(checkBulkGpsPush?.warning_no_location?.length ?? 0) > 0 && - (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - - - - - {formatMessage( - MESSAGES.noGpsForSomeInstaces, - )} - - - - - - {formatMessage(MESSAGES.seeAll)} - - - - - - - )} - {(checkBulkGpsPush?.warning_overwrite?.length ?? 0) > 0 && - (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 && ( - - - - - {formatMessage( - MESSAGES.someOrgUnitsHasAlreadyGps, - )} - - - - - - {formatMessage(MESSAGES.seeAll)} - - - - - - - )} + + 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 + } + message={MESSAGES.noGpsForSomeInstaces} + linkTo="url" + approveCondition={approveSubmissionNoHasGps} + onApproveClick={() => onApprove('instanceNoGps')} + /> + + 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 + } + message={MESSAGES.someOrgUnitsHasAlreadyGps} + linkTo="url" + approveCondition={approveOrgUnitHasGps} + onApproveClick={() => onApprove('orgUnitHasGps')} + /> ); From ba208e4a74c300af75917c6319d7225cfe5a10be Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Fri, 20 Dec 2024 10:20:37 +0200 Subject: [PATCH 43/62] Move warning code into different file --- .../components/PushBulkGpsWarning.tsx | 60 +++++++++++++++++++ .../components/PushGpsDialogComponent.tsx | 56 ++--------------- 2 files changed, 65 insertions(+), 51 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx new file mode 100644 index 0000000000..53114365ea --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx @@ -0,0 +1,60 @@ +import { Button, Grid, Typography } from '@mui/material'; +import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import React, { FunctionComponent } from 'react'; +import MESSAGES from '../messages'; + +type Props = { + condition: boolean; + message: { defaultMessage: string; id: string }; + linkTo: string; + approveCondition: boolean; + onApproveClick: () => void; +}; +const PushBulkGpsWarning: FunctionComponent = ({ + condition, + message, + linkTo, + approveCondition, + onApproveClick, +}) => { + const { formatMessage } = useSafeIntl(); + if (!condition) return null; + + return ( + + + + + {formatMessage(message)} + + + + + + {formatMessage(MESSAGES.seeAll)} + + + + + + + ); +}; + +export default PushBulkGpsWarning; diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index cde93c7737..38ec59fb04 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -1,64 +1,18 @@ import React, { FunctionComponent, useState } from 'react'; -import { Button, Grid, Typography } from '@mui/material'; -import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import { Grid, Typography } from '@mui/material'; +import { useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; import { Instance } from '../types/instance'; import { Selection } from '../../orgUnits/types/selection'; import { useGetCheckBulkGpsPush } from '../hooks/useGetCheckBulkGpsPush'; +import PushBulkGpsWarning from './PushBulkGpsWarning'; type Props = { renderTrigger: (openDialog: boolean) => void; selection: Selection; }; -const WarningSection = ({ - condition, - message, - linkTo, - approveCondition, - onApproveClick, -}) => { - const { formatMessage } = useSafeIntl(); - if (!condition) return null; - - return ( - - - - - {formatMessage(message)} - - - - - - {formatMessage(MESSAGES.seeAll)} - - - - - - - ); -}; - const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger, selection, @@ -135,7 +89,7 @@ const PushGpsDialogComponent: FunctionComponent = ({ })} - 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 @@ -145,7 +99,7 @@ const PushGpsDialogComponent: FunctionComponent = ({ approveCondition={approveSubmissionNoHasGps} onApproveClick={() => onApprove('instanceNoGps')} /> - 0 && (checkBulkGpsPush?.error_ids?.length ?? 0) <= 0 From c640e5861d5e39643b3ac4bc88dcf7f12d44753a Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 20 Dec 2024 12:28:46 +0100 Subject: [PATCH 44/62] Added task instance_bulk_gps_push Refactored the bulk gps check in order to call it inside the task. Fixed typo in HasCreateOrgUnitPermission. --- hat/audit/models.py | 1 + iaso/api/instances.py | 81 ++-- iaso/api/org_units.py | 6 +- .../tasks/create/instance_bulk_gps_push.py | 33 ++ iaso/tasks/instance_bulk_gps_push.py | 76 ++++ iaso/tests/api/test_instances.py | 80 +++- .../tasks/test_instance_bulk_gps_push.py | 400 ++++++++++++++++++ iaso/urls.py | 2 + iaso/utils/models/common.py | 77 ++++ 9 files changed, 692 insertions(+), 64 deletions(-) create mode 100644 iaso/api/tasks/create/instance_bulk_gps_push.py create mode 100644 iaso/tasks/instance_bulk_gps_push.py create mode 100644 iaso/tests/tasks/test_instance_bulk_gps_push.py diff --git a/hat/audit/models.py b/hat/audit/models.py index f97f0e367b..c323d6e1a0 100644 --- a/hat/audit/models.py +++ b/hat/audit/models.py @@ -16,6 +16,7 @@ ORG_UNIT_API_BULK = "org_unit_api_bulk" GROUP_SET_API = "group_set_api" INSTANCE_API = "instance_api" +INSTANCE_API_BULK = "instance_api_bulk" FORM_API = "form_api" GPKG_IMPORT = "gpkg_import" CAMPAIGN_API = "campaign_api" diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 8f87b5a8ea..86eefd8d62 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -38,9 +38,10 @@ ) from iaso.utils import timestamp_to_datetime from iaso.utils.file_utils import get_file_type +from .org_units import HasCreateOrgUnitPermission from ..models.forms import CR_MODE_IF_REFERENCE_FORM -from ..utils.models.common import get_creator_name +from ..utils.models.common import get_creator_name, check_instance_bulk_gps_push from . import common from .comment import UserSerializerForComment from .common import ( @@ -91,7 +92,7 @@ def validate_period(self, value): class HasInstancePermission(permissions.BasePermission): def has_permission(self, request: Request, view): - if request.method == "POST": + if request.method == "POST": # to handle anonymous submissions sent by mobile return True return request.user.is_authenticated and ( @@ -112,6 +113,20 @@ def has_object_permission(self, request: Request, view, obj: Instance): return False +class HasInstanceBulkPermission(permissions.BasePermission): + """ + Designed for POST endpoints that are not designed to receive new submissions. + """ + + def has_permission(self, request: Request, view): + return request.user.is_authenticated and ( + request.user.has_perm(permission.FORMS) + or request.user.has_perm(permission.SUBMISSIONS) + or request.user.has_perm(permission.REGISTRY_WRITE) + or request.user.has_perm(permission.REGISTRY_READ) + ) + + class InstanceFileSerializer(serializers.Serializer): id = serializers.IntegerField(read_only=True) instance_id = serializers.IntegerField() @@ -605,7 +620,11 @@ def bulkdelete(self, request): status=201, ) - @action(detail=False, methods=["GET"], permission_classes=[permissions.IsAuthenticated, HasInstancePermission]) + @action( + detail=False, + methods=["GET"], + permission_classes=[permissions.IsAuthenticated, HasInstanceBulkPermission, HasCreateOrgUnitPermission], + ) def check_bulk_gps_push(self, request): # first, let's parse all parameters received from the URL select_all, selected_ids, unselected_ids = self._parse_check_bulk_gps_push_parameters(request.GET) @@ -628,48 +647,15 @@ def check_bulk_gps_push(self, request): else: instances_query = instances_query.exclude(pk__in=unselected_ids) - overwrite_ids = [] - no_location_ids = [] - org_units_to_instances_dict = {} - set_org_units_ids = set() + success, errors, warnings = check_instance_bulk_gps_push(instances_query) - for instance in instances_query: - if not instance.location: - no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit - continue - - org_unit = instance.org_unit - if org_unit.id in org_units_to_instances_dict: - # we can't push this instance's location since there was another instance linked to this OrgUnit - org_units_to_instances_dict[org_unit.id].append(instance.id) - continue - else: - org_units_to_instances_dict[org_unit.id] = [instance.id] - - set_org_units_ids.add(org_unit.id) - if org_unit.location or org_unit.geom: - overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location - continue - - # Before returning, we need to check if we've had multiple hits on an OrgUnit - error_same_org_unit_ids = self._check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + if not success: + errors["result"] = "errors" + return Response(errors, status=status.HTTP_400_BAD_REQUEST) - if len(error_same_org_unit_ids): - return Response( - {"result": "error", "error_ids": error_same_org_unit_ids}, - status=status.HTTP_400_BAD_REQUEST, - ) - - if len(no_location_ids) or len(overwrite_ids): - dict_response = { - "result": "warnings", - } - if len(no_location_ids): - dict_response["warning_no_location"] = no_location_ids - if len(overwrite_ids): - dict_response["warning_overwrite"] = overwrite_ids - - return Response(dict_response, status=status.HTTP_200_OK) + if warnings: + warnings["result"] = "warnings" + return Response(warnings, status=status.HTTP_200_OK) return Response( { @@ -680,7 +666,7 @@ def check_bulk_gps_push(self, request): def _parse_check_bulk_gps_push_parameters(self, query_parameters): raw_select_all = query_parameters.get("select_all", True) - select_all = raw_select_all not in ["false", "False", "0"] + select_all = raw_select_all not in ["false", "False", "0", 0, False] raw_selected_ids = query_parameters.get("selected_ids", None) if raw_selected_ids: @@ -696,13 +682,6 @@ def _parse_check_bulk_gps_push_parameters(self, query_parameters): return select_all, selected_ids, unselected_ids - def _check_bulk_gps_repeated_org_units(self, org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: - error_instance_ids = [] - for _, instance_ids in org_units_to_instance_ids.items(): - if len(instance_ids) >= 2: - error_instance_ids.extend(instance_ids) - return error_instance_ids - QUERY = """ select DATE_TRUNC('month', COALESCE(iaso_instance.source_created_at, iaso_instance.created_at)) as month, (select name from iaso_form where id = iaso_instance.form_id) as form_name, diff --git a/iaso/api/org_units.py b/iaso/api/org_units.py index 45623ff7de..e28568a20f 100644 --- a/iaso/api/org_units.py +++ b/iaso/api/org_units.py @@ -32,7 +32,7 @@ # noinspection PyMethodMayBeStatic -class HasCreateOrUnitPermission(permissions.BasePermission): +class HasCreateOrgUnitPermission(permissions.BasePermission): def has_permission(self, request, view): if not request.user.is_authenticated: return False @@ -614,7 +614,9 @@ def get_date(self, date: str) -> Union[datetime.date, None]: pass return None - @action(detail=False, methods=["POST"], permission_classes=[permissions.IsAuthenticated, HasCreateOrUnitPermission]) + @action( + detail=False, methods=["POST"], permission_classes=[permissions.IsAuthenticated, HasCreateOrgUnitPermission] + ) def create_org_unit(self, request): """This endpoint is used by the React frontend""" errors = [] diff --git a/iaso/api/tasks/create/instance_bulk_gps_push.py b/iaso/api/tasks/create/instance_bulk_gps_push.py new file mode 100644 index 0000000000..3ba956269c --- /dev/null +++ b/iaso/api/tasks/create/instance_bulk_gps_push.py @@ -0,0 +1,33 @@ +from rest_framework import viewsets, permissions, status +from rest_framework.response import Response + +from iaso.api.instances import HasInstanceBulkPermission +from iaso.api.org_units import HasCreateOrgUnitPermission +from iaso.api.tasks import TaskSerializer +from iaso.tasks.instance_bulk_gps_push import instance_bulk_gps_push + + +class InstanceBulkGpsPushViewSet(viewsets.ViewSet): + """Bulk push gps location from Instances to their related OrgUnit. + + This task will override existing location on OrgUnits and might set `None` if the Instance doesn't have any location. + Calling this endpoint implies that the InstanceViewSet.check_bulk_gps_push() method has been called before and has returned no error. + """ + + permission_classes = [permissions.IsAuthenticated, HasInstanceBulkPermission, HasCreateOrgUnitPermission] + + def create(self, request): + raw_select_all = request.data.get("select_all", True) + select_all = raw_select_all not in [False, "false", "False", "0", 0] + selected_ids = request.data.get("selected_ids", []) + unselected_ids = request.data.get("unselected_ids", []) + + user = self.request.user + + task = instance_bulk_gps_push( + select_all=select_all, selected_ids=selected_ids, unselected_ids=unselected_ids, user=user + ) + return Response( + {"task": TaskSerializer(instance=task).data}, + status=status.HTTP_201_CREATED, + ) diff --git a/iaso/tasks/instance_bulk_gps_push.py b/iaso/tasks/instance_bulk_gps_push.py new file mode 100644 index 0000000000..53ac920d33 --- /dev/null +++ b/iaso/tasks/instance_bulk_gps_push.py @@ -0,0 +1,76 @@ +from copy import deepcopy +from logging import getLogger +from time import time +from typing import Optional, List + +from django.contrib.auth.models import User +from django.db import transaction + +from beanstalk_worker import task_decorator +from hat.audit import models as audit_models +from iaso.models import Task, Instance +from iaso.utils.gis import convert_2d_point_to_3d +from iaso.utils.models.common import check_instance_bulk_gps_push + +logger = getLogger(__name__) + + +def push_single_instance_gps_to_org_unit(user: Optional[User], instance: Instance): + org_unit = instance.org_unit + original_copy = deepcopy(org_unit) + org_unit.location = convert_2d_point_to_3d(instance.location) if instance.location else None + org_unit.save() + if not original_copy.location: + logger.info(f"updating {org_unit.name} {org_unit.id} with {org_unit.location}") + else: + logger.info( + f"updating {org_unit.name} {org_unit.id} - overwriting {original_copy.location} with {org_unit.location}" + ) + audit_models.log_modification(original_copy, org_unit, source=audit_models.INSTANCE_API_BULK, user=user) + + +@task_decorator(task_name="instance_bulk_gps_push") +def instance_bulk_gps_push( + select_all: bool, + selected_ids: List[int], + unselected_ids: List[int], + task: Task, +): + """Background Task to bulk push instance gps to org units. + + /!\ Danger: calling this task without having received a successful response from the check_bulk_gps_push + endpoint will have unexpected results that might cause data loss. + """ + start = time() + task.report_progress_and_stop_if_killed(progress_message="Searching for Instances for pushing gps data") + + user = task.launcher + + queryset = Instance.non_deleted_objects.get_queryset().filter_for_user(user) + queryset = queryset.select_related("org_unit") + + if not select_all: + queryset = queryset.filter(pk__in=selected_ids) + else: + queryset = queryset.exclude(pk__in=unselected_ids) + + if not queryset: + raise Exception("No matching instances found") + + # Checking if any gps push can be performed with what was requested + success, errors, _ = check_instance_bulk_gps_push(queryset) + if not success: + raise Exception("Cannot proceed with the gps push due to errors: %s" % errors) + + total = queryset.count() + + with transaction.atomic(): + for index, instance in enumerate(queryset.iterator()): + res_string = "%.2f sec, processed %i instances" % (time() - start, index) + task.report_progress_and_stop_if_killed(progress_message=res_string, end_value=total, progress_value=index) + push_single_instance_gps_to_org_unit( + user, + instance, + ) + + task.report_success(message="%d modified" % total) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index e7defdfddc..11cf8256f3 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -39,7 +39,11 @@ def setUpTestData(cls): cls.sw_version = sw_version cls.yoda = cls.create_user_with_profile( - username="yoda", last_name="Da", first_name="Yo", account=star_wars, permissions=["iaso_submissions"] + username="yoda", + last_name="Da", + first_name="Yo", + account=star_wars, + permissions=["iaso_submissions", "iaso_org_units"], ) cls.guest = cls.create_user_with_profile(username="guest", account=star_wars, permissions=["iaso_submissions"]) cls.supervisor = cls.create_user_with_profile( @@ -72,10 +76,15 @@ def setUpTestData(cls): version=sw_version, ) cls.jedi_council_endor = m.OrgUnit.objects.create( - name="Endor Jedi Council", source_ref="jedi_council_endor_ref" + name="Endor Jedi Council", + source_ref="jedi_council_endor_ref", + version=sw_version, ) cls.jedi_council_endor_region = m.OrgUnit.objects.create( - name="Endor Region Jedi Council", parent=cls.jedi_council_endor, source_ref="jedi_council_endor_region_ref" + name="Endor Region Jedi Council", + parent=cls.jedi_council_endor, + source_ref="jedi_council_endor_region_ref", + version=sw_version, ) cls.project = m.Project.objects.create( @@ -1963,8 +1972,8 @@ def test_check_bulk_push_gps_select_all_ok(self): response_json = response.json() self.assertEqual(response_json["result"], "success") - def test_check_bulk_push_gps_select_all_error(self): - # setting gps data for instances that were not deleted + def test_check_bulk_push_gps_select_all_error_same_org_unit(self): + # changing location for some instances to have multiple hits on multiple org_units self.instance_1.org_unit = self.jedi_council_endor self.instance_2.org_unit = self.jedi_council_endor new_location = Point(1, 2, 3) @@ -1973,14 +1982,61 @@ def test_check_bulk_push_gps_select_all_error(self): instance.save() instance.refresh_from_db() + # Let's delete some instances, the result will be the same + for instance in [self.instance_6, self.instance_8]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + self.client.force_authenticate(self.yoda) response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual( + response_json["error_same_org_unit"], + [self.instance_1.id, self.instance_2.id, self.instance_3.id, self.instance_4.id, self.instance_5.id], + ) + + def test_check_bulk_push_gps_select_all_error_read_only_source(self): + # Making the source read only + self.sw_source.read_only = True + self.sw_source.save() + + # Changing some instance.org_unit so that all the results don't appear only in "error_same_org_unit" + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + for instance in [self.instance_2, self.instance_3, self.instance_8]: + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "errors") + # instance_6 included because it's the first one with the remaining org_unit and the queryset has a default order of "-id" + self.assertCountEqual( + response_json["error_read_only_source"], + [self.instance_8.id, self.instance_2.id, self.instance_3.id, self.instance_6.id], + ) def test_check_bulk_push_gps_select_all_warning_no_location(self): + # Changing some instance.org_unit so that all the results don't appear only in "error_same_org_unit" + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + for instance in [self.instance_2, self.instance_3, self.instance_8]: + instance.save() + instance.refresh_from_db() + + # Let's delete some instances to avoid getting "error_same_org-unit" + for instance in [self.instance_4, self.instance_5, self.instance_6, self.instance_8]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + self.client.force_authenticate(self.yoda) response = self.client.get(f"/api/instances/check_bulk_gps_push/") self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -2095,8 +2151,10 @@ def test_check_bulk_push_gps_selected_ids_error(self): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() # All these Instances target the same OrgUnit, so it's impossible to push gps data - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual( + response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id, self.instance_3.id] + ) def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) @@ -2254,8 +2312,8 @@ def test_check_bulk_push_gps_unselected_ids_error(self): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual(response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) diff --git a/iaso/tests/tasks/test_instance_bulk_gps_push.py b/iaso/tests/tasks/test_instance_bulk_gps_push.py new file mode 100644 index 0000000000..0910baeb1f --- /dev/null +++ b/iaso/tests/tasks/test_instance_bulk_gps_push.py @@ -0,0 +1,400 @@ +from django.contrib.auth.models import Permission +from django.contrib.contenttypes.models import ContentType +from django.contrib.gis.geos import Point +from rest_framework import status + +from hat.menupermissions import models as am +from iaso import models as m +from iaso.models import Task, QUEUED +from iaso.tests.tasks.task_api_test_case import TaskAPITestCase + + +class InstanceBulkPushGpsAPITestCase(TaskAPITestCase): + BASE_URL = "/api/tasks/create/instancebulkgpspush/" + + @classmethod + def setUpTestData(cls): + # Preparing account, data source, project, users... + cls.account, cls.data_source, cls.source_version, cls.project = cls.create_account_datasource_version_project( + "source", "account", "project" + ) + cls.user, cls.anon_user, cls.user_no_perms = cls.create_base_users( + cls.account, ["iaso_submissions", "iaso_org_units"] + ) + + # Preparing org units & locations + cls.org_unit_type = m.OrgUnitType.objects.create(name="Org unit type", short_name="OUT") + cls.org_unit_type.projects.add(cls.project) + cls.org_unit_no_location = m.OrgUnit.objects.create( + name="No location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + cls.default_location = Point(x=4, y=50, z=100, srid=4326) + cls.other_location = Point(x=2, y=-50, z=100, srid=4326) + cls.org_unit_with_default_location = m.OrgUnit.objects.create( + name="Default location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + location=cls.default_location, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + cls.org_unit_with_other_location = m.OrgUnit.objects.create( + name="Other location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + location=cls.other_location, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + + # Preparing instances - all linked to org_unit_without_location + cls.form = m.Form.objects.create(name="form", period_type=m.MONTH, single_per_period=True) + cls.instance_without_location = cls.create_form_instance( + form=cls.form, + period="202001", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="noLoc", + ) + cls.instance_with_default_location = cls.create_form_instance( + form=cls.form, + period="202002", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="defaultLoc", + location=cls.default_location, + ) + cls.instance_with_other_location = cls.create_form_instance( + form=cls.form, + period="202003", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="otherLoc", + location=cls.other_location, + ) + + def test_ok(self): + """POST /api/tasks/create/instancebulkgpspush/ without any error nor warning""" + + # Setting up one more instance and orgunit + new_org_unit = m.OrgUnit.objects.create( + name="new org unit", + org_unit_type=self.org_unit_type, + validation_status=m.OrgUnit.VALIDATION_VALID, + version=self.source_version, + source_ref="new org unit", + ) + new_instance = m.Instance.objects.create( + org_unit=new_org_unit, + form=self.form, + period="202004", + project=self.project, + created_by=self.user, + export_id="instance4", + ) + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_without_location.id, new_instance.id], + }, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_201_CREATED) + + def test_not_logged_in(self): + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_401_UNAUTHORIZED) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_no_permission_instances(self): + """POST /api/tasks/create/instancebulkgpspush/ without instances permissions""" + # Adding org unit permission to user + content_type = ContentType.objects.get_for_model(am.CustomPermissionSupport) + self.user_no_perms.user_permissions.add( + Permission.objects.filter(codename="iaso_org_units", content_type=content_type).first().id + ) + + self.client.force_authenticate(self.user_no_perms) + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_no_permission_org_units(self): + """POST /api/tasks/create/instancebulkgpspush/ without orgunit permissions""" + # Adding instances permission to user + content_type = ContentType.objects.get_for_model(am.CustomPermissionSupport) + self.user_no_perms.user_permissions.add( + Permission.objects.filter(codename="iaso_submissions", content_type=content_type).first().id + ) + + self.client.force_authenticate(self.user_no_perms) + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_instance_ids_wrong_account(self): + """POST /api/tasks/create/instancebulkgpspush/ with instance IDs from another account""" + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) + new_user = self.create_user_with_profile( + username="new user", account=new_account, permissions=["iaso_submissions", "iaso_org_units"] + ) + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + _ = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + location=Point(1, 2, 3, 4326), + ) + + self.client.force_authenticate(new_user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_without_location.id, self.instance_with_default_location.id], + }, + format="json", + ) + + # Task is successfully created but will fail once it starts + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, new_user) + + # Let's run the task to see the error + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + self.assertEqual( + task.result["message"], "No matching instances found" + ) # Because the instance IDs are from another account + + # Making sure that nothing changed in both accounts + self.assertIsNone(new_org_unit.location) + self.assertIsNone(self.org_unit_no_location.location) + self.assertEqual(self.org_unit_with_default_location.location, self.default_location) + + def test_overwrite_existing_location(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that overwrite existing org unit locations""" + # Setting a new location for both org_units + location = Point(42, 69, 420, 4326) + for org_unit in [self.org_unit_with_default_location, self.org_unit_with_other_location]: + org_unit.location = location + org_unit.save() + + # Linking both instances to these org_units + self.instance_with_default_location.org_unit = self.org_unit_with_default_location + self.instance_with_default_location.save() + self.instance_with_other_location.org_unit = self.org_unit_with_other_location + self.instance_with_other_location.save() + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_with_default_location.id, self.instance_with_other_location.id], + }, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be a success + self.runAndValidateTask(task, "SUCCESS") + + self.org_unit_with_default_location.refresh_from_db() + self.org_unit_with_other_location.refresh_from_db() + self.assertEqualLocations(self.org_unit_with_default_location.location, self.default_location) + self.assertEqualLocations(self.org_unit_with_other_location.location, self.other_location) + self.assertIsNone(self.org_unit_no_location.location) + + def test_no_location(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that don't have any location defined""" + # Let's create another instance without a location, but this time it's linked to self.org_unit_with_default_location + _ = m.Instance.objects.create( + form=self.form, + period="202001", + org_unit=self.org_unit_with_default_location, + project=self.project, + created_by=self.user, + export_id="noLoc", + ) + + self.client.force_authenticate(self.user) + # For a change, let's select everything, but remove the two instances with a location + # (= it's the same as electing both instances without location) + response = self.client.post( + self.BASE_URL, + data={ + "unselected_ids": [self.instance_with_default_location.id, self.instance_with_other_location.id], + }, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be a success + self.runAndValidateTask(task, "SUCCESS") + + self.org_unit_with_default_location.refresh_from_db() + self.org_unit_no_location.refresh_from_db() + self.assertIsNone(self.org_unit_with_default_location.location) # Got overwritten by None + self.assertIsNone(self.org_unit_no_location.location) # Still None + self.assertEqualLocations(self.org_unit_with_other_location.location, self.other_location) # Not updated + + def test_multiple_updates_same_org_unit(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that target the same orgunit""" + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_same_org_unit", result) + for instance in [ + self.instance_without_location, + self.instance_with_other_location, + self.instance_with_default_location, + ]: + self.assertIn(str(instance.id), result) + + def test_read_only_data_source(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that target orgunits which are part of a read-only data source""" + self.data_source.read_only = True + self.data_source.save() + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_read_only_source", result) + for instance in [ + self.instance_without_location, + self.instance_with_other_location, + self.instance_with_default_location, + ]: + self.assertIn(str(instance.id), result) + + def test_all_errors(self): + """POST /api/tasks/create/instancebulkgpspush/ all errors are triggered""" + # Preparing a new read-only data source + new_data_source = m.DataSource.objects.create(name="new data source", read_only=True) + new_version = m.SourceVersion.objects.create(data_source=new_data_source, number=2) + new_data_source.projects.set([self.project]) + new_org_unit = m.OrgUnit.objects.create( + name="new org unit", + org_unit_type=self.org_unit_type, + validation_status=m.OrgUnit.VALIDATION_VALID, + version=new_version, + source_ref="new org unit", + ) + new_instance = m.Instance.objects.create( + org_unit=new_org_unit, + form=self.form, + period="202004", + project=self.project, + created_by=self.user, + export_id="instance4", + location=self.default_location, + ) + + # Changing this org unit so that it does not trigger error_same_org_unit + self.instance_with_default_location.org_unit = self.org_unit_with_default_location + self.instance_with_default_location.save() + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_read_only_source", result) + self.assertIn("error_same_org_unit", result) + for instance in [self.instance_without_location, self.instance_with_other_location, new_instance]: + self.assertIn(str(instance.id), result) # Instead, we should probably check in which error they end up + self.assertNotIn(str(self.instance_with_default_location.id), result) + + def test_task_kill(self): + """Launch the task and then kill it + Note this actually doesn't work if it's killed while in the transaction part. + """ + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + data = self.assertJSONResponse(response, status.HTTP_201_CREATED) + self.assertValidTaskAndInDB(data["task"]) + + task = Task.objects.get(id=data["task"]["id"]) + task.should_be_killed = True + task.save() + + self.runAndValidateTask(task, "KILLED") + + def assertEqualLocations(self, point_1: Point, point_2: Point): + self.assertEqual(point_1.x, point_2.x) + self.assertEqual(point_1.y, point_2.y) + self.assertEqual(point_1.z, point_2.z) + self.assertEqual(point_1.srid, point_2.srid) diff --git a/iaso/urls.py b/iaso/urls.py index 16e080f16e..d5110c70c9 100644 --- a/iaso/urls.py +++ b/iaso/urls.py @@ -94,6 +94,7 @@ from .api.tasks import TaskSourceViewSet from .api.tasks.create.export_mobile_setup import ExportMobileSetupViewSet from .api.tasks.create.import_gpkg import ImportGPKGViewSet +from .api.tasks.create.instance_bulk_gps_push import InstanceBulkGpsPushViewSet from .api.tasks.create.org_unit_bulk_location_set import OrgUnitsBulkLocationSet from .api.user_roles import UserRolesViewSet from .api.workflows.changes import WorkflowChangeViewSet @@ -169,6 +170,7 @@ router.register(r"tasks/create/orgunitsbulklocationset", OrgUnitsBulkLocationSet, basename="orgunitsbulklocationset") router.register(r"tasks/create/importgpkg", ImportGPKGViewSet, basename="importgpkg") router.register(r"tasks/create/exportmobilesetup", ExportMobileSetupViewSet, basename="exportmobilesetup") +router.register(r"tasks/create/instancebulkgpspush", InstanceBulkGpsPushViewSet, basename="instancebulkgpspush") router.register(r"tasks", TaskSourceViewSet, basename="tasks") router.register(r"comments", CommentViewSet, basename="comments") router.register(r"entities", EntityViewSet, basename="entity") diff --git a/iaso/utils/models/common.py b/iaso/utils/models/common.py index 77dd3c031c..541ee3b53a 100644 --- a/iaso/utils/models/common.py +++ b/iaso/utils/models/common.py @@ -1,3 +1,7 @@ +from typing import Dict, List + +from django.db.models import QuerySet + from iaso.models.base import User @@ -23,3 +27,76 @@ def get_org_unit_parents_ref(field_name, org_unit, parent_source_ref_field_names if parent_ref: return f"iaso#{parent_ref}" return None + + +def check_instance_bulk_gps_push(queryset: QuerySet) -> (bool, Dict[str, List[int]], Dict[str, List[int]]): + """ + Determines if there are any warnings or errors if the given Instances were to push their own location to their OrgUnit. + + There are 2 types of warnings: + - warning_no_location: if an Instance doesn't have any location + - warning_overwrite: if the Instance's OrgUnit already has a location + The gps push can be performed even if there are any warnings, keeping in mind the consequences. + + There are 2 types of errors: + - error_same_org_unit: if there are multiple Instances in the given queryset that share the same OrgUnit + - error_read_only_source: if any Instance's OrgUnit is part of a read-only DataSource + The gps push cannot be performed if there are any errors. + """ + # Variables used for warnings + set_org_units_ids = set() + overwrite_ids = [] + no_location_ids = [] + + # Variables used for errors + org_units_to_instances_dict = {} + read_only_data_sources = [] + + for instance in queryset: + # First, let's check for potential errors + org_unit = instance.org_unit + if org_unit.id in org_units_to_instances_dict: + # we can't push this instance's location since there was another instance linked to this OrgUnit + org_units_to_instances_dict[org_unit.id].append(instance.id) + continue + else: + org_units_to_instances_dict[org_unit.id] = [instance.id] + + if org_unit.version and org_unit.version.data_source.read_only: + read_only_data_sources.append(instance.id) + continue + + # Then, let's check for potential warnings + if not instance.location: + no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit + continue + + set_org_units_ids.add(org_unit.id) + if org_unit.location or org_unit.geom: + overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location + continue + + # Before returning, we need to check if we've had multiple hits on an OrgUnit + error_same_org_unit_ids = _check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + + success: bool = not read_only_data_sources and not error_same_org_unit_ids + errors = {} + if read_only_data_sources: + errors["error_read_only_source"] = read_only_data_sources + if error_same_org_unit_ids: + errors["error_same_org_unit"] = error_same_org_unit_ids + warnings = {} + if no_location_ids: + warnings["warning_no_location"] = no_location_ids + if overwrite_ids: + warnings["warning_overwrite"] = overwrite_ids + + return success, errors, warnings + + +def _check_bulk_gps_repeated_org_units(org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: + error_instance_ids = [] + for _, instance_ids in org_units_to_instance_ids.items(): + if len(instance_ids) >= 2: + error_instance_ids.extend(instance_ids) + return error_instance_ids From 2215a66fcc822ead31dc2ba519bc0481119911ee Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Fri, 20 Dec 2024 15:29:58 +0200 Subject: [PATCH 45/62] launch task --- .../components/PushGpsDialogComponent.tsx | 53 ++++++++++++++----- .../hooks/useInstanceBulkgpspush.tsx | 23 ++++++++ 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx index 38ec59fb04..e54e884d0b 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushGpsDialogComponent.tsx @@ -1,12 +1,14 @@ -import React, { FunctionComponent, useState } from 'react'; +import React, { FunctionComponent, useCallback, useState } from 'react'; import { Grid, Typography } from '@mui/material'; -import { useSafeIntl } from 'bluesquare-components'; +import { useRedirectTo, useSafeIntl } from 'bluesquare-components'; import ConfirmCancelDialogComponent from '../../../components/dialogs/ConfirmCancelDialogComponent'; import MESSAGES from '../messages'; import { Instance } from '../types/instance'; import { Selection } from '../../orgUnits/types/selection'; import { useGetCheckBulkGpsPush } from '../hooks/useGetCheckBulkGpsPush'; import PushBulkGpsWarning from './PushBulkGpsWarning'; +import { useInstanceBulkgpspush } from '../hooks/useInstanceBulkgpspush'; +import { baseUrls } from '../../../constants/urls'; type Props = { renderTrigger: (openDialog: boolean) => void; @@ -21,16 +23,41 @@ const PushGpsDialogComponent: FunctionComponent = ({ useState(false); const [approveSubmissionNoHasGps, setApproveSubmissionNoHasGps] = useState(false); - const onConfirm = closeDialog => { - return null; - }; + const { mutateAsync: bulkgpspush } = useInstanceBulkgpspush(); + const select_all = selection.selectAll; + const selected_ids = selection.selectedItems; + const unselected_ids = selection.unSelectedItems; + const instancebulkgpspush = useCallback(async () => { + await bulkgpspush({ + select_all, + selected_ids: selected_ids.map(item => item.id), + unselected_ids: unselected_ids.map(item => item.id), + }); + }, [bulkgpspush, select_all, selected_ids, unselected_ids]); + + const onConfirm = useCallback( + async closeDialog => { + await instancebulkgpspush(); + closeDialog(); + }, + [instancebulkgpspush], + ); + const redirectTo = useRedirectTo(); + const onConfirmAndSeeTask = useCallback( + async closeDialog => { + await instancebulkgpspush(); + closeDialog(); + redirectTo(baseUrls.tasks, { + order: '-created_at', + }); + }, + [instancebulkgpspush, redirectTo], + ); const { data: checkBulkGpsPush, isError } = useGetCheckBulkGpsPush({ - selected_ids: selection.selectedItems.map(item => item.id).join(','), - select_all: selection.selectAll, - unselected_ids: selection.unSelectedItems - .map(item => item.id) - .join(','), + selected_ids: selected_ids.map(item => item.id).join(','), + select_all, + unselected_ids: unselected_ids.map(item => item.id).join(','), }); const onClosed = () => { @@ -66,14 +93,16 @@ const PushGpsDialogComponent: FunctionComponent = ({ renderTrigger(openDialog)} titleMessage={title} - onConfirm={onConfirm} + onConfirm={closeDialog => onConfirm(closeDialog)} confirmMessage={MESSAGES.launch} onClosed={onClosed} cancelMessage={MESSAGES.cancel} maxWidth="sm" additionalButton additionalMessage={MESSAGES.goToCurrentTask} - onAdditionalButtonClick={onConfirm} + onAdditionalButtonClick={closeDialog => + onConfirmAndSeeTask(closeDialog) + } allowConfimAdditionalButton > diff --git a/hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx b/hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx new file mode 100644 index 0000000000..c8c4d9123d --- /dev/null +++ b/hat/assets/js/apps/Iaso/domains/instances/hooks/useInstanceBulkgpspush.tsx @@ -0,0 +1,23 @@ +import { UseMutationResult } from 'react-query'; +import { postRequest } from '../../../libs/Api'; +import { useSnackMutation } from '../../../libs/apiHooks'; + +type BulkGpsPush = { + select_all: boolean; + selected_ids: string[]; + unselected_ids: string[]; +}; +export const useInstanceBulkgpspush = (): UseMutationResult => { + return useSnackMutation({ + mutationFn: (data: BulkGpsPush) => { + const { select_all, selected_ids, unselected_ids } = data; + + return postRequest('/api/tasks/create/instancebulkgpspush/', { + select_all, + selected_ids, + unselected_ids, + }); + }, + showSucessSnackBar: false, + }); +}; From c8355aa89ce4fcb79b01611c0ae8717f3b895d8e Mon Sep 17 00:00:00 2001 From: HAKIZIMANA Franck Date: Fri, 20 Dec 2024 16:09:33 +0200 Subject: [PATCH 46/62] remove see all link --- .../components/PushBulkGpsWarning.tsx | 8 +-- .../components/PushGpsDialogComponent.tsx | 57 +++++++++++++++---- .../hooks/useGetCheckBulkGpsPush.tsx | 3 +- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx index 53114365ea..eefcc47c00 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/PushBulkGpsWarning.tsx @@ -1,19 +1,17 @@ import { Button, Grid, Typography } from '@mui/material'; -import { LinkWithLocation, useSafeIntl } from 'bluesquare-components'; +import { useSafeIntl } from 'bluesquare-components'; import React, { FunctionComponent } from 'react'; import MESSAGES from '../messages'; type Props = { condition: boolean; message: { defaultMessage: string; id: string }; - linkTo: string; approveCondition: boolean; onApproveClick: () => void; }; const PushBulkGpsWarning: FunctionComponent = ({ condition, message, - linkTo, approveCondition, onApproveClick, }) => { @@ -36,11 +34,11 @@ const PushBulkGpsWarning: FunctionComponent = ({ - + {/* {formatMessage(MESSAGES.seeAll)} - + */}